<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package serviceaccount

import (
	"bytes"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rsa"
	"encoding/pem"
	"errors"
	"fmt"
	"io/ioutil"

	"k8s.io/api/core/v1"
	"k8s.io/apiserver/pkg/authentication/authenticator"
	apiserverserviceaccount "k8s.io/apiserver/pkg/authentication/serviceaccount"
	"k8s.io/apiserver/pkg/authentication/user"
	"k8s.io/client-go/util/cert"

	jwt "github.com/dgrijalva/jwt-go"
	"github.com/golang/glog"
)

const (
	Issuer = "kubernetes/serviceaccount"

	SubjectClaim            = "sub"
	IssuerClaim             = "iss"
	ServiceAccountNameClaim = "kubernetes.io/serviceaccount/service-account.name"
	ServiceAccountUIDClaim  = "kubernetes.io/serviceaccount/service-account.uid"
	SecretNameClaim         = "kubernetes.io/serviceaccount/secret.name"
	NamespaceClaim          = "kubernetes.io/serviceaccount/namespace"
)

// ServiceAccountTokenGetter defines functions to retrieve a named service account and secret
type ServiceAccountTokenGetter interface {
	GetServiceAccount(namespace, name string) (*v1.ServiceAccount, error)
	GetSecret(namespace, name string) (*v1.Secret, error)
}

type TokenGenerator interface {
	// GenerateToken generates a token which will identify the given ServiceAccount.
	// The returned token will be stored in the given (and yet-unpersisted) Secret.
	GenerateToken(serviceAccount v1.ServiceAccount, secret v1.Secret) (string, error)
}

// ReadPrivateKey is a helper function for reading a private key from a PEM-encoded file
func ReadPrivateKey(file string) (interface{}, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	key, err := cert.ParsePrivateKeyPEM(data)
	if err != nil {
		return nil, fmt.Errorf("error reading private key file %s: %v", file, err)
	}
	return key, nil
}

// ReadPublicKeys is a helper function for reading an array of rsa.PublicKey or ecdsa.PublicKey from a PEM-encoded file.
// Reads public keys from both public and private key files.
func ReadPublicKeys(file string) ([]interface{}, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	keys, err := ReadPublicKeysFromPEM(data)
	if err != nil {
		return nil, fmt.Errorf("error reading public key file %s: %v", file, err)
	}
	return keys, nil
}

// ReadPublicKeysFromPEM is a helper function for reading an array of rsa.PublicKey or ecdsa.PublicKey from a PEM-encoded byte array.
// Reads public keys from both public and private key files.
func ReadPublicKeysFromPEM(data []byte) ([]interface{}, error) {
	var block *pem.Block
	keys := []interface{}{}
	for {
		// read the next block
		block, data = pem.Decode(data)
		if block == nil {
			break
		}

		// get PEM bytes for just this block
		blockData := pem.EncodeToMemory(block)
		if privateKey, err := jwt.ParseRSAPrivateKeyFromPEM(blockData); err == nil {
			keys = append(keys, &privateKey.PublicKey)
			continue
		}
		if publicKey, err := jwt.ParseRSAPublicKeyFromPEM(blockData); err == nil {
			keys = append(keys, publicKey)
			continue
		}

		if privateKey, err := jwt.ParseECPrivateKeyFromPEM(blockData); err == nil {
			keys = append(keys, &privateKey.PublicKey)
			continue
		}
		if publicKey, err := jwt.ParseECPublicKeyFromPEM(blockData); err == nil {
			keys = append(keys, publicKey)
			continue
		}

		// tolerate non-key PEM blocks for backwards compatibility
		// originally, only the first PEM block was parsed and expected to be a key block
	}

	if len(keys) == 0 {
		return nil, fmt.Errorf("data does not contain a valid RSA or ECDSA key")
	}
	return keys, nil
}

// JWTTokenGenerator returns a TokenGenerator that generates signed JWT tokens, using the given privateKey.
// privateKey is a PEM-encoded byte array of a private RSA key.
// JWTTokenAuthenticator()
func JWTTokenGenerator(privateKey interface{}) TokenGenerator {
	return &jwtTokenGenerator{privateKey}
}

type jwtTokenGenerator struct {
	privateKey interface{}
}

func (j *jwtTokenGenerator) GenerateToken(serviceAccount v1.ServiceAccount, secret v1.Secret) (string, error) {
	var method jwt.SigningMethod
	switch privateKey := j.privateKey.(type) {
	case *rsa.PrivateKey:
		method = jwt.SigningMethodRS256
	case *ecdsa.PrivateKey:
		switch privateKey.Curve {
		case elliptic.P256():
			method = jwt.SigningMethodES256
		case elliptic.P384():
			method = jwt.SigningMethodES384
		case elliptic.P521():
			method = jwt.SigningMethodES512
		default:
			return "", fmt.Errorf("unknown private key curve, must be 256, 384, or 521")
		}
	default:
		return "", fmt.Errorf("unknown private key type %T, must be *rsa.PrivateKey or *ecdsa.PrivateKey", j.privateKey)
	}

	token := jwt.New(method)

	claims, _ := token.Claims.(jwt.MapClaims)

	// Identify the issuer
	claims[IssuerClaim] = Issuer

	// Username
	claims[SubjectClaim] = apiserverserviceaccount.MakeUsername(serviceAccount.Namespace, serviceAccount.Name)

	// Persist enough structured info for the authenticator to be able to look up the service account and secret
	claims[NamespaceClaim] = serviceAccount.Namespace
	claims[ServiceAccountNameClaim] = serviceAccount.Name
	claims[ServiceAccountUIDClaim] = serviceAccount.UID
	claims[SecretNameClaim] = secret.Name

	// Sign and get the complete encoded token as a string
	return token.SignedString(j.privateKey)
}

// JWTTokenAuthenticator authenticates tokens as JWT tokens produced by JWTTokenGenerator
// Token signatures are verified using each of the given public keys until one works (allowing key rotation)
// If lookup is true, the service account and secret referenced as claims inside the token are retrieved and verified with the provided ServiceAccountTokenGetter
func JWTTokenAuthenticator(keys []interface{}, lookup bool, getter ServiceAccountTokenGetter) authenticator.Token {
	return &jwtTokenAuthenticator{keys, lookup, getter}
}

type jwtTokenAuthenticator struct {
	keys   []interface{}
	lookup bool
	getter ServiceAccountTokenGetter
}

var errMismatchedSigningMethod = errors.New("invalid signing method")

func (j *jwtTokenAuthenticator) AuthenticateToken(token string) (user.Info, bool, error) {
	var validationError error

	for i, key := range j.keys {
		// Attempt to verify with each key until we find one that works
		parsedToken, err := jwt.Parse(token, func(token *jwt.Token) (interface{}, error) {
			switch token.Method.(type) {
			case *jwt.SigningMethodRSA:
				if _, ok := key.(*rsa.PublicKey); ok {
					return key, nil
				}
				return nil, errMismatchedSigningMethod
			case *jwt.SigningMethodECDSA:
				if _, ok := key.(*ecdsa.PublicKey); ok {
					return key, nil
				}
				return nil, errMismatchedSigningMethod
			default:
				return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
			}
		})

		if err != nil {
			switch err := err.(type) {
			case *jwt.ValidationError:
				if (err.Errors & jwt.ValidationErrorMalformed) != 0 {
					// Not a JWT, no point in continuing
					return nil, false, nil
				}

				if (err.Errors & jwt.ValidationErrorSignatureInvalid) != 0 {
					// Signature error, perhaps one of the other keys will verify the signature
					// If not, we want to return this error
					glog.V(4).Infof("Signature error (key %d): %v", i, err)
					validationError = err
					continue
				}

				// This key doesn't apply to the given signature type
				// Perhaps one of the other keys will verify the signature
				// If not, we want to return this error
				if err.Inner == errMismatchedSigningMethod {
					glog.V(4).Infof("Mismatched key type (key %d): %v", i, err)
					validationError = err
					continue
				}
			}

			// Other errors should just return as errors
			return nil, false, err
		}

		// If we get here, we have a token with a recognized signature

		claims, _ := parsedToken.Claims.(jwt.MapClaims)

		// Make sure we issued the token
		iss, _ := claims[IssuerClaim].(string)
		if iss != Issuer {
			return nil, false, nil
		}

		// Make sure the claims we need exist
		sub, _ := claims[SubjectClaim].(string)
		if len(sub) == 0 {
			return nil, false, errors.New("sub claim is missing")
		}
		namespace, _ := claims[NamespaceClaim].(string)
		if len(namespace) == 0 {
			return nil, false, errors.New("namespace claim is missing")
		}
		secretName, _ := claims[SecretNameClaim].(string)
		if len(namespace) == 0 {
			return nil, false, errors.New("secretName claim is missing")
		}
		serviceAccountName, _ := claims[ServiceAccountNameClaim].(string)
		if len(serviceAccountName) == 0 {
			return nil, false, errors.New("serviceAccountName claim is missing")
		}
		serviceAccountUID, _ := claims[ServiceAccountUIDClaim].(string)
		if len(serviceAccountUID) == 0 {
			return nil, false, errors.New("serviceAccountUID claim is missing")
		}

		subjectNamespace, subjectName, err := apiserverserviceaccount.SplitUsername(sub)
		if err != nil || subjectNamespace != namespace || subjectName != serviceAccountName {
			return nil, false, errors.New("sub claim is invalid")
		}

		if j.lookup {
			// Make sure token hasn't been invalidated by deletion of the secret
			secret, err := j.getter.GetSecret(namespace, secretName)
			if err != nil {
				glog.V(4).Infof("Could not retrieve token %s/%s for service account %s/%s: %v", namespace, secretName, namespace, serviceAccountName, err)
				return nil, false, errors.New("Token has been invalidated")
			}
			if bytes.Compare(secret.Data[v1.ServiceAccountTokenKey], []byte(token)) != 0 {
				glog.V(4).Infof("Token contents no longer matches %s/%s for service account %s/%s", namespace, secretName, namespace, serviceAccountName)
				return nil, false, errors.New("Token does not match server's copy")
			}

			// Make sure service account still exists (name and UID)
			serviceAccount, err := j.getter.GetServiceAccount(namespace, serviceAccountName)
			if err != nil {
				glog.V(4).Infof("Could not retrieve service account %s/%s: %v", namespace, serviceAccountName, err)
				return nil, false, err
			}
			if string(serviceAccount.UID) != serviceAccountUID {
				glog.V(4).Infof("Service account UID no longer matches %s/%s: %q != %q", namespace, serviceAccountName, string(serviceAccount.UID), serviceAccountUID)
				return nil, false, fmt.Errorf("ServiceAccount UID (%s) does not match claim (%s)", serviceAccount.UID, serviceAccountUID)
			}
		}

		return UserInfo(namespace, serviceAccountName, serviceAccountUID), true, nil
	}

	return nil, false, validationError
}
<commit_msg>don't accept delete tokens that are waiting to be reaped<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package serviceaccount

import (
	"bytes"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rsa"
	"encoding/pem"
	"errors"
	"fmt"
	"io/ioutil"

	"k8s.io/api/core/v1"
	"k8s.io/apiserver/pkg/authentication/authenticator"
	apiserverserviceaccount "k8s.io/apiserver/pkg/authentication/serviceaccount"
	"k8s.io/apiserver/pkg/authentication/user"
	"k8s.io/client-go/util/cert"

	jwt "github.com/dgrijalva/jwt-go"
	"github.com/golang/glog"
)

const (
	Issuer = "kubernetes/serviceaccount"

	SubjectClaim            = "sub"
	IssuerClaim             = "iss"
	ServiceAccountNameClaim = "kubernetes.io/serviceaccount/service-account.name"
	ServiceAccountUIDClaim  = "kubernetes.io/serviceaccount/service-account.uid"
	SecretNameClaim         = "kubernetes.io/serviceaccount/secret.name"
	NamespaceClaim          = "kubernetes.io/serviceaccount/namespace"
)

// ServiceAccountTokenGetter defines functions to retrieve a named service account and secret
type ServiceAccountTokenGetter interface {
	GetServiceAccount(namespace, name string) (*v1.ServiceAccount, error)
	GetSecret(namespace, name string) (*v1.Secret, error)
}

type TokenGenerator interface {
	// GenerateToken generates a token which will identify the given ServiceAccount.
	// The returned token will be stored in the given (and yet-unpersisted) Secret.
	GenerateToken(serviceAccount v1.ServiceAccount, secret v1.Secret) (string, error)
}

// ReadPrivateKey is a helper function for reading a private key from a PEM-encoded file
func ReadPrivateKey(file string) (interface{}, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	key, err := cert.ParsePrivateKeyPEM(data)
	if err != nil {
		return nil, fmt.Errorf("error reading private key file %s: %v", file, err)
	}
	return key, nil
}

// ReadPublicKeys is a helper function for reading an array of rsa.PublicKey or ecdsa.PublicKey from a PEM-encoded file.
// Reads public keys from both public and private key files.
func ReadPublicKeys(file string) ([]interface{}, error) {
	data, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	keys, err := ReadPublicKeysFromPEM(data)
	if err != nil {
		return nil, fmt.Errorf("error reading public key file %s: %v", file, err)
	}
	return keys, nil
}

// ReadPublicKeysFromPEM is a helper function for reading an array of rsa.PublicKey or ecdsa.PublicKey from a PEM-encoded byte array.
// Reads public keys from both public and private key files.
func ReadPublicKeysFromPEM(data []byte) ([]interface{}, error) {
	var block *pem.Block
	keys := []interface{}{}
	for {
		// read the next block
		block, data = pem.Decode(data)
		if block == nil {
			break
		}

		// get PEM bytes for just this block
		blockData := pem.EncodeToMemory(block)
		if privateKey, err := jwt.ParseRSAPrivateKeyFromPEM(blockData); err == nil {
			keys = append(keys, &privateKey.PublicKey)
			continue
		}
		if publicKey, err := jwt.ParseRSAPublicKeyFromPEM(blockData); err == nil {
			keys = append(keys, publicKey)
			continue
		}

		if privateKey, err := jwt.ParseECPrivateKeyFromPEM(blockData); err == nil {
			keys = append(keys, &privateKey.PublicKey)
			continue
		}
		if publicKey, err := jwt.ParseECPublicKeyFromPEM(blockData); err == nil {
			keys = append(keys, publicKey)
			continue
		}

		// tolerate non-key PEM blocks for backwards compatibility
		// originally, only the first PEM block was parsed and expected to be a key block
	}

	if len(keys) == 0 {
		return nil, fmt.Errorf("data does not contain a valid RSA or ECDSA key")
	}
	return keys, nil
}

// JWTTokenGenerator returns a TokenGenerator that generates signed JWT tokens, using the given privateKey.
// privateKey is a PEM-encoded byte array of a private RSA key.
// JWTTokenAuthenticator()
func JWTTokenGenerator(privateKey interface{}) TokenGenerator {
	return &jwtTokenGenerator{privateKey}
}

type jwtTokenGenerator struct {
	privateKey interface{}
}

func (j *jwtTokenGenerator) GenerateToken(serviceAccount v1.ServiceAccount, secret v1.Secret) (string, error) {
	var method jwt.SigningMethod
	switch privateKey := j.privateKey.(type) {
	case *rsa.PrivateKey:
		method = jwt.SigningMethodRS256
	case *ecdsa.PrivateKey:
		switch privateKey.Curve {
		case elliptic.P256():
			method = jwt.SigningMethodES256
		case elliptic.P384():
			method = jwt.SigningMethodES384
		case elliptic.P521():
			method = jwt.SigningMethodES512
		default:
			return "", fmt.Errorf("unknown private key curve, must be 256, 384, or 521")
		}
	default:
		return "", fmt.Errorf("unknown private key type %T, must be *rsa.PrivateKey or *ecdsa.PrivateKey", j.privateKey)
	}

	token := jwt.New(method)

	claims, _ := token.Claims.(jwt.MapClaims)

	// Identify the issuer
	claims[IssuerClaim] = Issuer

	// Username
	claims[SubjectClaim] = apiserverserviceaccount.MakeUsername(serviceAccount.Namespace, serviceAccount.Name)

	// Persist enough structured info for the authenticator to be able to look up the service account and secret
	claims[NamespaceClaim] = serviceAccount.Namespace
	claims[ServiceAccountNameClaim] = serviceAccount.Name
	claims[ServiceAccountUIDClaim] = serviceAccount.UID
	claims[SecretNameClaim] = secret.Name

	// Sign and get the complete encoded token as a string
	return token.SignedString(j.privateKey)
}

// JWTTokenAuthenticator authenticates tokens as JWT tokens produced by JWTTokenGenerator
// Token signatures are verified using each of the given public keys until one works (allowing key rotation)
// If lookup is true, the service account and secret referenced as claims inside the token are retrieved and verified with the provided ServiceAccountTokenGetter
func JWTTokenAuthenticator(keys []interface{}, lookup bool, getter ServiceAccountTokenGetter) authenticator.Token {
	return &jwtTokenAuthenticator{keys, lookup, getter}
}

type jwtTokenAuthenticator struct {
	keys   []interface{}
	lookup bool
	getter ServiceAccountTokenGetter
}

var errMismatchedSigningMethod = errors.New("invalid signing method")

func (j *jwtTokenAuthenticator) AuthenticateToken(token string) (user.Info, bool, error) {
	var validationError error

	for i, key := range j.keys {
		// Attempt to verify with each key until we find one that works
		parsedToken, err := jwt.Parse(token, func(token *jwt.Token) (interface{}, error) {
			switch token.Method.(type) {
			case *jwt.SigningMethodRSA:
				if _, ok := key.(*rsa.PublicKey); ok {
					return key, nil
				}
				return nil, errMismatchedSigningMethod
			case *jwt.SigningMethodECDSA:
				if _, ok := key.(*ecdsa.PublicKey); ok {
					return key, nil
				}
				return nil, errMismatchedSigningMethod
			default:
				return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
			}
		})

		if err != nil {
			switch err := err.(type) {
			case *jwt.ValidationError:
				if (err.Errors & jwt.ValidationErrorMalformed) != 0 {
					// Not a JWT, no point in continuing
					return nil, false, nil
				}

				if (err.Errors & jwt.ValidationErrorSignatureInvalid) != 0 {
					// Signature error, perhaps one of the other keys will verify the signature
					// If not, we want to return this error
					glog.V(4).Infof("Signature error (key %d): %v", i, err)
					validationError = err
					continue
				}

				// This key doesn't apply to the given signature type
				// Perhaps one of the other keys will verify the signature
				// If not, we want to return this error
				if err.Inner == errMismatchedSigningMethod {
					glog.V(4).Infof("Mismatched key type (key %d): %v", i, err)
					validationError = err
					continue
				}
			}

			// Other errors should just return as errors
			return nil, false, err
		}

		// If we get here, we have a token with a recognized signature

		claims, _ := parsedToken.Claims.(jwt.MapClaims)

		// Make sure we issued the token
		iss, _ := claims[IssuerClaim].(string)
		if iss != Issuer {
			return nil, false, nil
		}

		// Make sure the claims we need exist
		sub, _ := claims[SubjectClaim].(string)
		if len(sub) == 0 {
			return nil, false, errors.New("sub claim is missing")
		}
		namespace, _ := claims[NamespaceClaim].(string)
		if len(namespace) == 0 {
			return nil, false, errors.New("namespace claim is missing")
		}
		secretName, _ := claims[SecretNameClaim].(string)
		if len(namespace) == 0 {
			return nil, false, errors.New("secretName claim is missing")
		}
		serviceAccountName, _ := claims[ServiceAccountNameClaim].(string)
		if len(serviceAccountName) == 0 {
			return nil, false, errors.New("serviceAccountName claim is missing")
		}
		serviceAccountUID, _ := claims[ServiceAccountUIDClaim].(string)
		if len(serviceAccountUID) == 0 {
			return nil, false, errors.New("serviceAccountUID claim is missing")
		}

		subjectNamespace, subjectName, err := apiserverserviceaccount.SplitUsername(sub)
		if err != nil || subjectNamespace != namespace || subjectName != serviceAccountName {
			return nil, false, errors.New("sub claim is invalid")
		}

		if j.lookup {
			// Make sure token hasn't been invalidated by deletion of the secret
			secret, err := j.getter.GetSecret(namespace, secretName)
			if err != nil {
				glog.V(4).Infof("Could not retrieve token %s/%s for service account %s/%s: %v", namespace, secretName, namespace, serviceAccountName, err)
				return nil, false, errors.New("Token has been invalidated")
			}
			if secret.DeletionTimestamp != nil {
				glog.V(4).Infof("Token is deleted and awaiting removal: %s/%s for service account %s/%s", namespace, secretName, namespace, serviceAccountName)
				return nil, false, errors.New("Token has been invalidated")
			}
			if bytes.Compare(secret.Data[v1.ServiceAccountTokenKey], []byte(token)) != 0 {
				glog.V(4).Infof("Token contents no longer matches %s/%s for service account %s/%s", namespace, secretName, namespace, serviceAccountName)
				return nil, false, errors.New("Token does not match server's copy")
			}

			// Make sure service account still exists (name and UID)
			serviceAccount, err := j.getter.GetServiceAccount(namespace, serviceAccountName)
			if err != nil {
				glog.V(4).Infof("Could not retrieve service account %s/%s: %v", namespace, serviceAccountName, err)
				return nil, false, err
			}
			if serviceAccount.DeletionTimestamp != nil {
				glog.V(4).Infof("Service account has been deleted %s/%s", namespace, serviceAccountName)
				return nil, false, fmt.Errorf("ServiceAccount %s/%s has been deleted", namespace, serviceAccountName)
			}
			if string(serviceAccount.UID) != serviceAccountUID {
				glog.V(4).Infof("Service account UID no longer matches %s/%s: %q != %q", namespace, serviceAccountName, string(serviceAccount.UID), serviceAccountUID)
				return nil, false, fmt.Errorf("ServiceAccount UID (%s) does not match claim (%s)", serviceAccount.UID, serviceAccountUID)
			}
		}

		return UserInfo(namespace, serviceAccountName, serviceAccountUID), true, nil
	}

	return nil, false, validationError
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sync

import (
	"context"
	"fmt"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/watch"
	"github.com/bmatcuk/doublestar"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var (
	// WorkingDir is here for testing
	WorkingDir = docker.RetrieveWorkingDir
)

type Syncer interface {
	Sync(context.Context, *Item) error
}

type syncMap map[string][]string

type Item struct {
	Image  string
	Copy   map[string][]string
	Delete map[string][]string
}

func NewItem(a *latest.Artifact, e watch.Events, builds []build.Artifact, insecureRegistries map[string]bool) (*Item, error) {
	// If there are no changes, short circuit and don't sync anything
	if !e.HasChanged() || a.Sync == nil || len(a.Sync.Manual) == 0 {
		return nil, nil
	}

	tag := latestTag(a.ImageName, builds)
	if tag == "" {
		return nil, fmt.Errorf("could not find latest tag for image %s in builds: %v", a.ImageName, builds)
	}

	containerWd, err := WorkingDir(tag, insecureRegistries)
	if err != nil {
		return nil, errors.Wrapf(err, "retrieving working dir for %s", tag)
	}

	toCopy, err := intersect(a.Workspace, containerWd, a.Sync.Manual, append(e.Added, e.Modified...))
	if err != nil {
		return nil, errors.Wrap(err, "intersecting sync map and added, modified files")
	}

	toDelete, err := intersect(a.Workspace, containerWd, a.Sync.Manual, e.Deleted)
	if err != nil {
		return nil, errors.Wrap(err, "intersecting sync map and deleted files")
	}

	// Something went wrong, don't sync, rebuild.
	if toCopy == nil || toDelete == nil {
		return nil, nil
	}

	return &Item{
		Image:  tag,
		Copy:   toCopy,
		Delete: toDelete,
	}, nil
}

func latestTag(image string, builds []build.Artifact) string {
	for _, build := range builds {
		if build.ImageName == image {
			return build.Tag
		}
	}
	return ""
}

func intersect(contextWd, containerWd string, syncRules []*latest.SyncRule, files []string) (syncMap, error) {
	ret := make(syncMap)
	for _, f := range files {
		relPath, err := filepath.Rel(contextWd, f)
		if err != nil {
			return nil, errors.Wrapf(err, "changed file %s can't be found relative to context %s", f, contextWd)
		}

		dsts, err := matchSyncRules(syncRules, relPath, containerWd)
		if err != nil {
			return nil, err
		}

		if len(dsts) == 0 {
			logrus.Infof("Changed file %s does not match any sync pattern. Skipping sync", relPath)
			return nil, nil
		}

		ret[f] = dsts
	}
	return ret, nil
}

func matchSyncRules(syncRules []*latest.SyncRule, relPath, containerWd string) ([]string, error) {
	dsts := make([]string, 0, 1)
	for _, r := range syncRules {
		matches, err := doublestar.PathMatch(filepath.FromSlash(r.Src), relPath)
		if err != nil {
			return nil, errors.Wrapf(err, "pattern error for %s", relPath)
		}

		if !matches {
			continue
		}

		wd := ""
		if !path.IsAbs(r.Dest) {
			// Convert relative destinations to absolute via the working dir in the container.
			wd = containerWd
		}

		// Map the paths as a tree from the prefix.
		subPath := strings.TrimPrefix(filepath.ToSlash(relPath), r.Strip)
		dsts = append(dsts, path.Join(wd, r.Dest, subPath))
	}
	return dsts, nil
}

func Perform(ctx context.Context, image string, files syncMap, cmdFn func(context.Context, v1.Pod, v1.Container, map[string][]string) []*exec.Cmd, namespaces []string) error {
	if len(files) == 0 {
		return nil
	}

	client, err := kubernetes.Client()
	if err != nil {
		return errors.Wrap(err, "getting k8s client")
	}

	numSynced := 0
	for _, ns := range namespaces {
		pods, err := client.CoreV1().Pods(ns).List(meta_v1.ListOptions{})
		if err != nil {
			return errors.Wrap(err, "getting pods for namespace "+ns)
		}

		for _, p := range pods.Items {
			for _, c := range p.Spec.Containers {
				if c.Image != image {
					continue
				}

				cmds := cmdFn(ctx, p, c, files)
				for _, cmd := range cmds {
					if _, err := util.RunCmdOut(cmd); err != nil {
						return err
					}
					numSynced++
				}
			}
		}
	}

	if numSynced == 0 {
		return errors.New("didn't sync any files")
	}

	return nil
}
<commit_msg>Use goroutine for sync (#2378)<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sync

import (
	"context"
	"fmt"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/watch"
	"github.com/bmatcuk/doublestar"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/sync/errgroup"
	v1 "k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var (
	// WorkingDir is here for testing
	WorkingDir = docker.RetrieveWorkingDir
)

type Syncer interface {
	Sync(context.Context, *Item) error
}

type syncMap map[string][]string

type Item struct {
	Image  string
	Copy   map[string][]string
	Delete map[string][]string
}

func NewItem(a *latest.Artifact, e watch.Events, builds []build.Artifact, insecureRegistries map[string]bool) (*Item, error) {
	// If there are no changes, short circuit and don't sync anything
	if !e.HasChanged() || a.Sync == nil || len(a.Sync.Manual) == 0 {
		return nil, nil
	}

	tag := latestTag(a.ImageName, builds)
	if tag == "" {
		return nil, fmt.Errorf("could not find latest tag for image %s in builds: %v", a.ImageName, builds)
	}

	containerWd, err := WorkingDir(tag, insecureRegistries)
	if err != nil {
		return nil, errors.Wrapf(err, "retrieving working dir for %s", tag)
	}

	toCopy, err := intersect(a.Workspace, containerWd, a.Sync.Manual, append(e.Added, e.Modified...))
	if err != nil {
		return nil, errors.Wrap(err, "intersecting sync map and added, modified files")
	}

	toDelete, err := intersect(a.Workspace, containerWd, a.Sync.Manual, e.Deleted)
	if err != nil {
		return nil, errors.Wrap(err, "intersecting sync map and deleted files")
	}

	// Something went wrong, don't sync, rebuild.
	if toCopy == nil || toDelete == nil {
		return nil, nil
	}

	return &Item{
		Image:  tag,
		Copy:   toCopy,
		Delete: toDelete,
	}, nil
}

func latestTag(image string, builds []build.Artifact) string {
	for _, build := range builds {
		if build.ImageName == image {
			return build.Tag
		}
	}
	return ""
}

func intersect(contextWd, containerWd string, syncRules []*latest.SyncRule, files []string) (syncMap, error) {
	ret := make(syncMap)
	for _, f := range files {
		relPath, err := filepath.Rel(contextWd, f)
		if err != nil {
			return nil, errors.Wrapf(err, "changed file %s can't be found relative to context %s", f, contextWd)
		}

		dsts, err := matchSyncRules(syncRules, relPath, containerWd)
		if err != nil {
			return nil, err
		}

		if len(dsts) == 0 {
			logrus.Infof("Changed file %s does not match any sync pattern. Skipping sync", relPath)
			return nil, nil
		}

		ret[f] = dsts
	}
	return ret, nil
}

func matchSyncRules(syncRules []*latest.SyncRule, relPath, containerWd string) ([]string, error) {
	dsts := make([]string, 0, 1)
	for _, r := range syncRules {
		matches, err := doublestar.PathMatch(filepath.FromSlash(r.Src), relPath)
		if err != nil {
			return nil, errors.Wrapf(err, "pattern error for %s", relPath)
		}

		if !matches {
			continue
		}

		wd := ""
		if !path.IsAbs(r.Dest) {
			// Convert relative destinations to absolute via the working dir in the container.
			wd = containerWd
		}

		// Map the paths as a tree from the prefix.
		subPath := strings.TrimPrefix(filepath.ToSlash(relPath), r.Strip)
		dsts = append(dsts, path.Join(wd, r.Dest, subPath))
	}
	return dsts, nil
}

func Perform(ctx context.Context, image string, files syncMap, cmdFn func(context.Context, v1.Pod, v1.Container, map[string][]string) []*exec.Cmd, namespaces []string) error {
	errs, ctx := errgroup.WithContext(ctx)
	if len(files) == 0 {
		return nil
	}

	client, err := kubernetes.Client()
	if err != nil {
		return errors.Wrap(err, "getting k8s client")
	}

	numSynced := 0
	for _, ns := range namespaces {
		pods, err := client.CoreV1().Pods(ns).List(meta_v1.ListOptions{})
		if err != nil {
			return errors.Wrap(err, "getting pods for namespace "+ns)
		}

		for _, p := range pods.Items {
			for _, c := range p.Spec.Containers {
				if c.Image != image {
					continue
				}

				cmds := cmdFn(ctx, p, c, files)
				for _, cmd := range cmds {
					errs.Go(func() error {
						_, err := util.RunCmdOut(cmd)
						return err
					})
					numSynced++
				}
			}
		}
	}

	if numSynced == 0 {
		return errors.New("didn't sync any files")
	}

	return errs.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/garyburd/redigo/redis"
	"github.com/gin-gonic/contrib/static"
	"github.com/gin-gonic/gin"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
)

const (
	SERVER_PORT string = ":80"
	CONFIG_PATH string = "/blog-data/private/config.json"
)

var (
	redisPool *redis.Pool
	config    *Config = &Config{Lang: []string{"en"}, Title: []string{"Title"}, PostsPerPage: 10}
)

func main() {

	installInitialData()
	err := LoadAndWatchConfig(config)
	if err != nil {
		log.Fatal(err)
	}

	redisPool = newRedisPool("blog-db:6379")

	// TODO: flush redis scripts

	legacyProxy := createLegacyProxy()

	router := gin.Default()
	router.LoadHTMLGlob("/blog-data/templates/*")
	router.Use(static.ServeRoot("/", "/blog-data/static"))

	router.Use(AttachConfig)
	router.Use(DefineLang)

	router.GET("/:slug/:id", func(c *gin.Context) {
		id := c.Param("id")

		post, err := postGet(id)
		if err != nil {
			c.AbortWithError(http.StatusInternalServerError, err)
			return
		}
		c.HTML(http.StatusOK, "post.tmpl", gin.H{
			"title": GetTitle(c),
			"post":  post,
		})
	})

	router.GET("/", func(c *gin.Context) {
		posts, err := postsList()
		if err != nil {
			c.AbortWithError(http.StatusInternalServerError, err)
			return
		}
		c.HTML(http.StatusOK, "default.tmpl", gin.H{
			"title": GetTitle(c),
			"posts": posts,
		})
	})

	router.Use(func(c *gin.Context) {
		legacyProxy.ServeHTTP(c.Writer, c.Request)
	})

	router.Run(SERVER_PORT)
}

// createLegacyProxy returns an http proxy to send
// requests to the legacy Node.js server
func createLegacyProxy() *httputil.ReverseProxy {
	u, err := url.Parse("http://blog-legacy")
	if err != nil {
		log.Fatalln(err)
	}
	return httputil.NewSingleHostReverseProxy(u)
}
<commit_msg>route fixes<commit_after>package main

import (
	"github.com/garyburd/redigo/redis"
	"github.com/gin-gonic/contrib/static"
	"github.com/gin-gonic/gin"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"regexp"
)

const (
	SERVER_PORT string = ":80"
	CONFIG_PATH string = "/blog-data/private/config.json"
)

var (
	redisPool *redis.Pool
	config    *Config = &Config{Lang: []string{"en"}, Title: []string{"Title"}, PostsPerPage: 10}
)

func main() {

	installInitialData()
	err := LoadAndWatchConfig(config)
	if err != nil {
		log.Fatal(err)
	}

	redisPool = newRedisPool("blog-db:6379")

	// TODO: flush redis scripts

	legacyProxy := createLegacyProxy()

	router := gin.Default()
	router.LoadHTMLGlob("/blog-data/templates/*")
	router.Use(static.ServeRoot("/", "/blog-data/static"))
	// legacy
	router.Use(static.ServeRoot("/", "/blog-data/public"))

	router.Use(AttachConfig)
	router.Use(DefineLang)

	router.GET("/:page/:param", func(c *gin.Context) {

		param := c.Param("param")
		paramIsID, err := regexp.MatchString("[0-9]+", param)
		if err != nil {
			c.AbortWithError(http.StatusInternalServerError, err)
			return
		}

		// if param is an ID, it means we want to display a post
		// (and page is just a slugged title)
		if paramIsID {
			post, err := postGet(param)
			if err != nil {
				c.AbortWithError(http.StatusInternalServerError, err)
				return
			}
			c.HTML(http.StatusOK, "post.tmpl", gin.H{
				"title": GetTitle(c),
				"post":  post,
			})
			return
		}

		// use Node.js legacy for other requests (like /admin routes)
		// TODO: Go implementation
		legacyProxy.ServeHTTP(c.Writer, c.Request)

	})

	router.GET("/", func(c *gin.Context) {
		posts, err := postsList()
		if err != nil {
			c.AbortWithError(http.StatusInternalServerError, err)
			return
		}
		c.HTML(http.StatusOK, "default.tmpl", gin.H{
			"title": GetTitle(c),
			"posts": posts,
		})
	})

	router.Use(func(c *gin.Context) {
		log.Println("proxy")
		legacyProxy.ServeHTTP(c.Writer, c.Request)
	})

	router.Run(SERVER_PORT)
}

// createLegacyProxy returns an http proxy to send
// requests to the legacy Node.js server
func createLegacyProxy() *httputil.ReverseProxy {
	u, err := url.Parse("http://blog-legacy")
	if err != nil {
		log.Fatalln(err)
	}
	return httputil.NewSingleHostReverseProxy(u)
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/validation"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/athena"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsAthenaDatabase() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsAthenaDatabaseCreate,
		Read:   resourceAwsAthenaDatabaseRead,
		Update: resourceAwsAthenaDatabaseUpdate,
		Delete: resourceAwsAthenaDatabaseDelete,

		Schema: map[string]*schema.Schema{
			"name": {
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: validation.StringMatch(regexp.MustCompile("^[_a-z0-9]+$"), "see https://docs.aws.amazon.com/athena/latest/ug/tables-databases-columns-names.html"),
			},
			"bucket": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"force_destroy": {
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
			"encryption_key": {
				Type:     schema.TypeList,
				Optional: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"id": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"type": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
		},
	}
}

func resourceAwsAthenaDatabaseCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).athenaconn

	e := d.Get("encryption_key").([]interface{})
	data := e[0].(map[string]interface{})
	keyType := data["type"].(string)
	keyID := data["id"].(string)

	resultConfig := athena.ResultConfiguration{
		OutputLocation: aws.String("s3://" + d.Get("bucket").(string)),
	}

	if len(keyType) > 0 {
		if strings.HasSuffix(keyType, "_KMS") && len(keyID) <= 0 {
			return fmt.Errorf("Key type %s requires a valid KMS key ID", keyType)
		}

		encryptionConfig := athena.EncryptionConfiguration{
			EncryptionOption: aws.String(keyType),
		}

		if len(keyID) > 0 {
			encryptionConfig.KmsKey = aws.String(keyID)
		}

		resultConfig.EncryptionConfiguration = &encryptionConfig
	}

	input := &athena.StartQueryExecutionInput{
		QueryString:         aws.String(fmt.Sprintf("create database `%s`;", d.Get("name").(string))),
		ResultConfiguration: &resultConfig,
	}

	resp, err := conn.StartQueryExecution(input)
	if err != nil {
		return err
	}

	if err := executeAndExpectNoRowsWhenCreate(*resp.QueryExecutionId, d, conn); err != nil {
		return err
	}
	d.SetId(d.Get("name").(string))
	return resourceAwsAthenaDatabaseRead(d, meta)
}

func resourceAwsAthenaDatabaseRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).athenaconn

	bucket := d.Get("bucket").(string)
	input := &athena.StartQueryExecutionInput{
		QueryString: aws.String(fmt.Sprint("show databases;")),
		ResultConfiguration: &athena.ResultConfiguration{
			OutputLocation: aws.String("s3://" + bucket),
		},
	}

	resp, err := conn.StartQueryExecution(input)
	if err != nil {
		return err
	}

	if err := executeAndExpectMatchingRow(*resp.QueryExecutionId, d.Get("name").(string), conn); err != nil {
		return err
	}
	return nil
}

func resourceAwsAthenaDatabaseUpdate(d *schema.ResourceData, meta interface{}) error {
	return resourceAwsAthenaDatabaseRead(d, meta)
}

func resourceAwsAthenaDatabaseDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).athenaconn

	name := d.Get("name").(string)
	bucket := d.Get("bucket").(string)

	queryString := fmt.Sprintf("drop database `%s`", name)
	if d.Get("force_destroy").(bool) {
		queryString += " cascade"
	}
	queryString += ";"

	input := &athena.StartQueryExecutionInput{
		QueryString: aws.String(queryString),
		ResultConfiguration: &athena.ResultConfiguration{
			OutputLocation: aws.String("s3://" + bucket),
		},
	}

	resp, err := conn.StartQueryExecution(input)
	if err != nil {
		return err
	}

	if err := executeAndExpectNoRowsWhenDrop(*resp.QueryExecutionId, d, conn); err != nil {
		return err
	}
	return nil
}

func executeAndExpectNoRowsWhenCreate(qeid string, d *schema.ResourceData, conn *athena.Athena) error {
	rs, err := queryExecutionResult(qeid, conn)
	if err != nil {
		return err
	}
	if len(rs.Rows) != 0 {
		return fmt.Errorf("Athena create database, unexpected query result: %s", flattenAthenaResultSet(rs))
	}
	return nil
}

func executeAndExpectMatchingRow(qeid string, dbName string, conn *athena.Athena) error {
	rs, err := queryExecutionResult(qeid, conn)
	if err != nil {
		return err
	}
	for _, row := range rs.Rows {
		for _, datum := range row.Data {
			if *datum.VarCharValue == dbName {
				return nil
			}
		}
	}
	return fmt.Errorf("Athena not found database: %s, query result: %s", dbName, flattenAthenaResultSet(rs))
}

func executeAndExpectNoRowsWhenDrop(qeid string, d *schema.ResourceData, conn *athena.Athena) error {
	rs, err := queryExecutionResult(qeid, conn)
	if err != nil {
		return err
	}
	if len(rs.Rows) != 0 {
		return fmt.Errorf("Athena drop database, unexpected query result: %s", flattenAthenaResultSet(rs))
	}
	return nil
}

func queryExecutionResult(qeid string, conn *athena.Athena) (*athena.ResultSet, error) {
	executionStateConf := &resource.StateChangeConf{
		Pending:    []string{athena.QueryExecutionStateQueued, athena.QueryExecutionStateRunning},
		Target:     []string{athena.QueryExecutionStateSucceeded},
		Refresh:    queryExecutionStateRefreshFunc(qeid, conn),
		Timeout:    10 * time.Minute,
		Delay:      3 * time.Second,
		MinTimeout: 3 * time.Second,
	}
	_, err := executionStateConf.WaitForState()
	if err != nil {
		return nil, err
	}

	qrinput := &athena.GetQueryResultsInput{
		QueryExecutionId: aws.String(qeid),
	}
	resp, err := conn.GetQueryResults(qrinput)
	if err != nil {
		return nil, err
	}
	return resp.ResultSet, nil
}

func queryExecutionStateRefreshFunc(qeid string, conn *athena.Athena) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		input := &athena.GetQueryExecutionInput{
			QueryExecutionId: aws.String(qeid),
		}
		out, err := conn.GetQueryExecution(input)
		if err != nil {
			return nil, "failed", err
		}
		status := out.QueryExecution.Status
		if *status.State == athena.QueryExecutionStateFailed &&
			status.StateChangeReason != nil {
			err = fmt.Errorf("reason: %s", *status.StateChangeReason)
		}
		return out, *out.QueryExecution.Status.State, err
	}
}

func flattenAthenaResultSet(rs *athena.ResultSet) string {
	ss := make([]string, 0)
	for _, row := range rs.Rows {
		for _, datum := range row.Data {
			ss = append(ss, *datum.VarCharValue)
		}
	}
	return strings.Join(ss, "\n")
}
<commit_msg>DRY the ResultConfiguration logic.<commit_after>package aws

import (
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/validation"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/athena"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsAthenaDatabase() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsAthenaDatabaseCreate,
		Read:   resourceAwsAthenaDatabaseRead,
		Update: resourceAwsAthenaDatabaseUpdate,
		Delete: resourceAwsAthenaDatabaseDelete,

		Schema: map[string]*schema.Schema{
			"name": {
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: validation.StringMatch(regexp.MustCompile("^[_a-z0-9]+$"), "see https://docs.aws.amazon.com/athena/latest/ug/tables-databases-columns-names.html"),
			},
			"bucket": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"force_destroy": {
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
			"encryption_key": {
				Type:     schema.TypeList,
				Optional: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"id": {
							Type:     schema.TypeString,
							Optional: true,
						},
						"type": {
							Type:     schema.TypeString,
							Required: true,
						},
					},
				},
			},
		},
	}
}

func getResultConfig(d *schema.ResourceData) (athena.ResultConfiguration, error) {
	e := d.Get("encryption_key").([]interface{})
	data := e[0].(map[string]interface{})
	keyType := data["type"].(string)
	keyID := data["id"].(string)

	resultConfig := athena.ResultConfiguration{
		OutputLocation: aws.String("s3://" + d.Get("bucket").(string)),
	}

	if len(keyType) > 0 {
		if strings.HasSuffix(keyType, "_KMS") && len(keyID) <= 0 {
			return fmt.Errorf("Key type %s requires a valid KMS key ID", keyType)
		}

		encryptionConfig := athena.EncryptionConfiguration{
			EncryptionOption: aws.String(keyType),
		}

		if len(keyID) > 0 {
			encryptionConfig.KmsKey = aws.String(keyID)
		}

		resultConfig.EncryptionConfiguration = &encryptionConfig
	}

	return resultConfig, nil
}

func resourceAwsAthenaDatabaseCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).athenaconn

	resultConfig, err := getResultConfig(d)
	if err != nil {
		return err
	}

	input := &athena.StartQueryExecutionInput{
		QueryString:         aws.String(fmt.Sprintf("create database `%s`;", d.Get("name").(string))),
		ResultConfiguration: &resultConfig,
	}

	resp, err := conn.StartQueryExecution(input)
	if err != nil {
		return err
	}

	if err := executeAndExpectNoRowsWhenCreate(*resp.QueryExecutionId, d, conn); err != nil {
		return err
	}
	d.SetId(d.Get("name").(string))
	return resourceAwsAthenaDatabaseRead(d, meta)
}

func resourceAwsAthenaDatabaseRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).athenaconn

	resultConfig, err := getResultConfig(d)
	if err != nil {
		return err
	}

	bucket := d.Get("bucket").(string)
	input := &athena.StartQueryExecutionInput{
		QueryString:         aws.String(fmt.Sprint("show databases;")),
		ResultConfiguration: &resultConfig,
	}

	resp, err := conn.StartQueryExecution(input)
	if err != nil {
		return err
	}

	if err := executeAndExpectMatchingRow(*resp.QueryExecutionId, d.Get("name").(string), conn); err != nil {
		return err
	}
	return nil
}

func resourceAwsAthenaDatabaseUpdate(d *schema.ResourceData, meta interface{}) error {
	return resourceAwsAthenaDatabaseRead(d, meta)
}

func resourceAwsAthenaDatabaseDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).athenaconn

	resultConfig, err := getResultConfig(d)
	if err != nil {
		return err
	}

	name := d.Get("name").(string)
	bucket := d.Get("bucket").(string)

	queryString := fmt.Sprintf("drop database `%s`", name)
	if d.Get("force_destroy").(bool) {
		queryString += " cascade"
	}
	queryString += ";"

	input := &athena.StartQueryExecutionInput{
		QueryString:         aws.String(queryString),
		ResultConfiguration: &resultConfig,
	}

	resp, err := conn.StartQueryExecution(input)
	if err != nil {
		return err
	}

	if err := executeAndExpectNoRowsWhenDrop(*resp.QueryExecutionId, d, conn); err != nil {
		return err
	}
	return nil
}

func executeAndExpectNoRowsWhenCreate(qeid string, d *schema.ResourceData, conn *athena.Athena) error {
	rs, err := queryExecutionResult(qeid, conn)
	if err != nil {
		return err
	}
	if len(rs.Rows) != 0 {
		return fmt.Errorf("Athena create database, unexpected query result: %s", flattenAthenaResultSet(rs))
	}
	return nil
}

func executeAndExpectMatchingRow(qeid string, dbName string, conn *athena.Athena) error {
	rs, err := queryExecutionResult(qeid, conn)
	if err != nil {
		return err
	}
	for _, row := range rs.Rows {
		for _, datum := range row.Data {
			if *datum.VarCharValue == dbName {
				return nil
			}
		}
	}
	return fmt.Errorf("Athena not found database: %s, query result: %s", dbName, flattenAthenaResultSet(rs))
}

func executeAndExpectNoRowsWhenDrop(qeid string, d *schema.ResourceData, conn *athena.Athena) error {
	rs, err := queryExecutionResult(qeid, conn)
	if err != nil {
		return err
	}
	if len(rs.Rows) != 0 {
		return fmt.Errorf("Athena drop database, unexpected query result: %s", flattenAthenaResultSet(rs))
	}
	return nil
}

func queryExecutionResult(qeid string, conn *athena.Athena) (*athena.ResultSet, error) {
	executionStateConf := &resource.StateChangeConf{
		Pending:    []string{athena.QueryExecutionStateQueued, athena.QueryExecutionStateRunning},
		Target:     []string{athena.QueryExecutionStateSucceeded},
		Refresh:    queryExecutionStateRefreshFunc(qeid, conn),
		Timeout:    10 * time.Minute,
		Delay:      3 * time.Second,
		MinTimeout: 3 * time.Second,
	}
	_, err := executionStateConf.WaitForState()
	if err != nil {
		return nil, err
	}

	qrinput := &athena.GetQueryResultsInput{
		QueryExecutionId: aws.String(qeid),
	}
	resp, err := conn.GetQueryResults(qrinput)
	if err != nil {
		return nil, err
	}
	return resp.ResultSet, nil
}

func queryExecutionStateRefreshFunc(qeid string, conn *athena.Athena) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		input := &athena.GetQueryExecutionInput{
			QueryExecutionId: aws.String(qeid),
		}
		out, err := conn.GetQueryExecution(input)
		if err != nil {
			return nil, "failed", err
		}
		status := out.QueryExecution.Status
		if *status.State == athena.QueryExecutionStateFailed &&
			status.StateChangeReason != nil {
			err = fmt.Errorf("reason: %s", *status.StateChangeReason)
		}
		return out, *out.QueryExecution.Status.State, err
	}
}

func flattenAthenaResultSet(rs *athena.ResultSet) string {
	ss := make([]string, 0)
	for _, row := range rs.Rows {
		for _, datum := range row.Data {
			ss = append(ss, *datum.VarCharValue)
		}
	}
	return strings.Join(ss, "\n")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html/template"
	"os"
	"path/filepath"
	"sort"
)

// NewBlogBuilder create a BlogBuilder instance that will build blog from
// with files in specified directory and configuration
func NewBlogBuilder(parser PostParser, conf Configuration, dir string) BlogBuilder {
	return BlogBuilder{dir, parser, map[string]*template.Template{}, conf}
}

// A BlogBuilder that generate static files from posts/layouts
type BlogBuilder struct {
	// where posts, layouts directory exist
	blogDir string

	// Parser that parse Post from post files
	postParser PostParser

	// map contains compiled template for later use
	templates map[string]*template.Template

	// configuration
	conf Configuration
}

// Build the blog, it will parse and copy post files into specified output
// directory
func (b BlogBuilder) Build(output string) error {
	os.RemoveAll(output)
	os.Mkdir(output, os.ModePerm)

	if err := b.compileTemplates(); err != nil {
		return fmt.Errorf("Fail to compile templates due to %v", err)
	}

	paths, err := b.getPostPaths()
	if err != nil {
		return err
	}

	posts := []Post{}
	for _, path := range paths {
		post := b.postParser.Parse(path)
		posts = append(posts, post)

		postOutputDir := filepath.Join(output, Prettify(post.Title()))
		err := b.generatePost(post, postOutputDir)
		if err != nil {
			return fmt.Errorf("Fail to generate post due to %v", err)
		}
	}

	sort.Sort(PostsByDateDesc{posts})

	if err := b.generateBlogIndex(posts, output); err != nil {
		return fmt.Errorf("Fail to generate blog index due to %v", err)
	}

	if err := b.copyAssets(output); err != nil {
		return fmt.Errorf("Fail to copy asset files due to %v ", err)
	}
	return nil
}

// getPostPaths will return a list of post directories in the blog folder
func (b BlogBuilder) getPostPaths() ([]string, error) {
	paths, err := filepath.Glob(filepath.Join(b.blogDir, PostsDirName, "*"))
	if err != nil {
		return nil, err
	}

	dirs := []string{}
	for _, path := range paths {
		info, err := os.Stat(path)
		if err != nil {
			return nil, err
		}
		if info.IsDir() {
			dirs = append(dirs, path)
		}
	}
	return dirs, nil
}

// compileTemplates compile all templates for later use
func (b BlogBuilder) compileTemplates() error {
	templates := map[string]string{
		"index": IndexTemplateName,
		"post":  PostTemplateName,
	}

	baseTemplatePath := filepath.Join(b.blogDir, LayoutsDirName, BaseTemplateName)
	for name, filename := range templates {
		templatePath := filepath.Join(b.blogDir, LayoutsDirName, filename)
		template, err := template.ParseFiles(baseTemplatePath, templatePath)
		if err != nil {
			return fmt.Errorf("Fail to parse template %s due to %v", filename, err)
		}
		b.templates[name] = template
	}
	return nil
}

// generatePost will generate files for specified post and put under specified
// output directory
func (b BlogBuilder) generatePost(post Post, output string) error {
	if err := os.Mkdir(output, os.ModePerm); err != nil {
		return err
	}

	if err := b.copyPostFiles(post.Folder(), output); err != nil {
		return err
	}

	index := filepath.Join(output, "index.html")
	file, err := os.Create(index)
	if err != nil {
		return fmt.Errorf("Unable to create file %s", index)
	}

	data := map[string]interface{}{
		"Content": template.HTML(post.HtmlContent()),
		"Title":   post.Title(),
		"Date":    post.Date(),
	}

	if err := b.templates["post"].Execute(file, data); err != nil {
		return err
	}
	return nil
}

// generateBlogIndex will generate blog index file under specified output path
func (b BlogBuilder) generateBlogIndex(posts []Post, output string) error {
	index := filepath.Join(output, "index.html")
	file, err := os.Create(index)
	if err != nil {
		return fmt.Errorf("Unable to create file %s", index)
	}

	data := map[string]interface{}{
		"Posts": posts,
	}

	if err := b.templates["index"].Execute(file, data); err != nil {
		return err
	}
	return nil
}

// copyPostFiles will copy all files under post dir into output path, but
// exclude post.md
func (b BlogBuilder) copyPostFiles(postDir string, output string) error {

	walkFn := func(path string, info os.FileInfo, err error) error {
		postFilePath := filepath.Join(postDir, PostFileName)
		if path == postDir || path == postFilePath {
			return nil
		}

		rel, err := filepath.Rel(postDir, path)
		if err != nil {
			return err
		}

		if info.IsDir() {
			return os.Mkdir(filepath.Join(output, rel), os.ModePerm)
		}
		return CopyFile(path, filepath.Join(output, rel))
	}

	return filepath.Walk(postDir, walkFn)
}

// copyAssets copy all files in assets to specified output directory
func (b BlogBuilder) copyAssets(output string) error {
	assetsDir := filepath.Join(b.blogDir, AssetsDirName)
	walkFn := func(path string, info os.FileInfo, err error) error {
		if path == assetsDir {
			return nil
		}

		rel, err := filepath.Rel(assetsDir, path)
		if err != nil {
			return err
		}

		if info.IsDir() {
			return os.Mkdir(filepath.Join(output, rel), os.ModePerm)
		}
		return CopyFile(path, filepath.Join(output, rel))
	}

	return filepath.Walk(assetsDir, walkFn)
}
<commit_msg>avoid unnecessary variable redeclaration<commit_after>package main

import (
	"fmt"
	"html/template"
	"os"
	"path/filepath"
	"sort"
)

// NewBlogBuilder create a BlogBuilder instance that will build blog from
// with files in specified directory and configuration
func NewBlogBuilder(parser PostParser, conf Configuration, dir string) BlogBuilder {
	return BlogBuilder{dir, parser, map[string]*template.Template{}, conf}
}

// A BlogBuilder that generate static files from posts/layouts
type BlogBuilder struct {
	// where posts, layouts directory exist
	blogDir string

	// Parser that parse Post from post files
	postParser PostParser

	// map contains compiled template for later use
	templates map[string]*template.Template

	// configuration
	conf Configuration
}

// Build the blog, it will parse and copy post files into specified output
// directory
func (b BlogBuilder) Build(output string) error {
	os.RemoveAll(output)
	os.Mkdir(output, os.ModePerm)

	if err := b.compileTemplates(); err != nil {
		return fmt.Errorf("Fail to compile templates due to %v", err)
	}

	paths, err := b.getPostPaths()
	if err != nil {
		return err
	}

	posts := []Post{}
	for _, path := range paths {
		post := b.postParser.Parse(path)
		posts = append(posts, post)

		postOutputDir := filepath.Join(output, Prettify(post.Title()))
		err := b.generatePost(post, postOutputDir)
		if err != nil {
			return fmt.Errorf("Fail to generate post due to %v", err)
		}
	}

	sort.Sort(PostsByDateDesc{posts})

	if err := b.generateBlogIndex(posts, output); err != nil {
		return fmt.Errorf("Fail to generate blog index due to %v", err)
	}

	if err := b.copyAssets(output); err != nil {
		return fmt.Errorf("Fail to copy asset files due to %v ", err)
	}
	return nil
}

// getPostPaths will return a list of post directories in the blog folder
func (b BlogBuilder) getPostPaths() ([]string, error) {
	paths, err := filepath.Glob(filepath.Join(b.blogDir, PostsDirName, "*"))
	if err != nil {
		return nil, err
	}

	dirs := []string{}
	for _, path := range paths {
		info, err := os.Stat(path)
		if err != nil {
			return nil, err
		}
		if info.IsDir() {
			dirs = append(dirs, path)
		}
	}
	return dirs, nil
}

// compileTemplates compile all templates for later use
func (b BlogBuilder) compileTemplates() error {
	templates := map[string]string{
		"index": IndexTemplateName,
		"post":  PostTemplateName,
	}

	baseTemplatePath := filepath.Join(b.blogDir, LayoutsDirName, BaseTemplateName)
	for name, filename := range templates {
		templatePath := filepath.Join(b.blogDir, LayoutsDirName, filename)
		template, err := template.ParseFiles(baseTemplatePath, templatePath)
		if err != nil {
			return fmt.Errorf("Fail to parse template %s due to %v", filename, err)
		}
		b.templates[name] = template
	}
	return nil
}

// generatePost will generate files for specified post and put under specified
// output directory
func (b BlogBuilder) generatePost(post Post, output string) error {
	if err := os.Mkdir(output, os.ModePerm); err != nil {
		return err
	}

	if err := b.copyPostFiles(post.Folder(), output); err != nil {
		return err
	}

	index := filepath.Join(output, "index.html")
	file, err := os.Create(index)
	if err != nil {
		return fmt.Errorf("Unable to create file %s", index)
	}

	data := map[string]interface{}{
		"Content": template.HTML(post.HtmlContent()),
		"Title":   post.Title(),
		"Date":    post.Date(),
	}

	if err := b.templates["post"].Execute(file, data); err != nil {
		return err
	}
	return nil
}

// generateBlogIndex will generate blog index file under specified output path
func (b BlogBuilder) generateBlogIndex(posts []Post, output string) error {
	index := filepath.Join(output, "index.html")
	file, err := os.Create(index)
	if err != nil {
		return fmt.Errorf("Unable to create file %s", index)
	}

	data := map[string]interface{}{
		"Posts": posts,
	}

	if err := b.templates["index"].Execute(file, data); err != nil {
		return err
	}
	return nil
}

// copyPostFiles will copy all files under post dir into output path, but
// exclude post.md
func (b BlogBuilder) copyPostFiles(postDir string, output string) error {

	postFilePath := filepath.Join(postDir, PostFileName)

	walkFn := func(path string, info os.FileInfo, err error) error {
		if path == postDir || path == postFilePath {
			return nil
		}

		rel, err := filepath.Rel(postDir, path)
		if err != nil {
			return err
		}

		if info.IsDir() {
			return os.Mkdir(filepath.Join(output, rel), os.ModePerm)
		}
		return CopyFile(path, filepath.Join(output, rel))
	}

	return filepath.Walk(postDir, walkFn)
}

// copyAssets copy all files in assets to specified output directory
func (b BlogBuilder) copyAssets(output string) error {
	assetsDir := filepath.Join(b.blogDir, AssetsDirName)
	walkFn := func(path string, info os.FileInfo, err error) error {
		if path == assetsDir {
			return nil
		}

		rel, err := filepath.Rel(assetsDir, path)
		if err != nil {
			return err
		}

		if info.IsDir() {
			return os.Mkdir(filepath.Join(output, rel), os.ModePerm)
		}
		return CopyFile(path, filepath.Join(output, rel))
	}

	return filepath.Walk(assetsDir, walkFn)
}
<|endoftext|>


<commit_before>package fleetmanager

import (
	"net"

	"github.com/Symantec/Dominator/lib/tags"
	proto "github.com/Symantec/Dominator/proto/hypervisor"
)

type ChangeMachineTagsRequest struct {
	Hostname string
	Tags     tags.Tags
}

type ChangeMachineTagsResponse struct {
	Error string
}

type GetHypervisorForVMRequest struct {
	IpAddress net.IP
}

type GetHypervisorForVMResponse struct {
	HypervisorAddress string // host:port
	Error             string
}

type GetMachineInfoRequest struct {
	Hostname string
}

type GetMachineInfoResponse struct {
	Error    string          `json:",omitempty"`
	Location string          `json:",omitempty"`
	Machine  Machine         `json:",omitempty"`
	Subnets  []*proto.Subnet `json:",omitempty"`
}

// The GetUpdates() RPC is fully streamed.
// The client sends a single GetUpdatesRequest message.
// The server sends a stream of Update messages.

type GetUpdatesRequest struct {
	Location   string
	MaxUpdates uint64 // Zero means infinite.
}

type Update struct {
	ChangedMachines []*Machine               `json:",omitempty"`
	ChangedVMs      map[string]*proto.VmInfo `json:",omitempty"` // Key: IPaddr
	DeletedMachines []string                 `json:",omitempty"` // Hostname
	DeletedVMs      []string                 `json:",omitempty"` // IPaddr
	Error           string                   `json:",omitempty"`
}

type HardwareAddr net.HardwareAddr

type ListHypervisorLocationsRequest struct {
	TopLocation string
}

type ListHypervisorLocationsResponse struct {
	Locations []string
	Error     string
}

type ListHypervisorsInLocationRequest struct {
	Location string
	SubnetId string
}

type ListHypervisorsInLocationResponse struct {
	HypervisorAddresses []string // host:port
	Error               string
}

type ListVMsInLocationRequest struct {
	Location string
}

// A stream of ListVMsInLocationResponse messages is sent, until either the
// length of the IpAddresses field is zero, or the Error field != "".
type ListVMsInLocationResponse struct {
	IpAddresses []net.IP
	Error       string
}

type Machine struct {
	NetworkEntry            `json:",omitempty"`
	IPMI                    NetworkEntry   `json:",omitempty"`
	OwnerGroups             []string       `json:",omitempty"`
	OwnerUsers              []string       `json:",omitempty"`
	SecondaryNetworkEntries []NetworkEntry `json:",omitempty"`
	Tags                    tags.Tags      `json:",omitempty"`
}

type NetworkEntry struct {
	Hostname       string       `json:",omitempty"`
	HostIpAddress  net.IP       `json:",omitempty"`
	HostMacAddress HardwareAddr `json:",omitempty"`
}
<commit_msg>Add proto/fleetmanager.MoveIpAddresses messages.<commit_after>package fleetmanager

import (
	"net"

	"github.com/Symantec/Dominator/lib/tags"
	proto "github.com/Symantec/Dominator/proto/hypervisor"
)

type ChangeMachineTagsRequest struct {
	Hostname string
	Tags     tags.Tags
}

type ChangeMachineTagsResponse struct {
	Error string
}

type GetHypervisorForVMRequest struct {
	IpAddress net.IP
}

type GetHypervisorForVMResponse struct {
	HypervisorAddress string // host:port
	Error             string
}

type GetMachineInfoRequest struct {
	Hostname string
}

type GetMachineInfoResponse struct {
	Error    string          `json:",omitempty"`
	Location string          `json:",omitempty"`
	Machine  Machine         `json:",omitempty"`
	Subnets  []*proto.Subnet `json:",omitempty"`
}

// The GetUpdates() RPC is fully streamed.
// The client sends a single GetUpdatesRequest message.
// The server sends a stream of Update messages.

type GetUpdatesRequest struct {
	Location   string
	MaxUpdates uint64 // Zero means infinite.
}

type Update struct {
	ChangedMachines []*Machine               `json:",omitempty"`
	ChangedVMs      map[string]*proto.VmInfo `json:",omitempty"` // Key: IPaddr
	DeletedMachines []string                 `json:",omitempty"` // Hostname
	DeletedVMs      []string                 `json:",omitempty"` // IPaddr
	Error           string                   `json:",omitempty"`
}

type HardwareAddr net.HardwareAddr

type ListHypervisorLocationsRequest struct {
	TopLocation string
}

type ListHypervisorLocationsResponse struct {
	Locations []string
	Error     string
}

type ListHypervisorsInLocationRequest struct {
	Location string
	SubnetId string
}

type ListHypervisorsInLocationResponse struct {
	HypervisorAddresses []string // host:port
	Error               string
}

type ListVMsInLocationRequest struct {
	Location string
}

// A stream of ListVMsInLocationResponse messages is sent, until either the
// length of the IpAddresses field is zero, or the Error field != "".
type ListVMsInLocationResponse struct {
	IpAddresses []net.IP
	Error       string
}

type Machine struct {
	NetworkEntry            `json:",omitempty"`
	IPMI                    NetworkEntry   `json:",omitempty"`
	OwnerGroups             []string       `json:",omitempty"`
	OwnerUsers              []string       `json:",omitempty"`
	SecondaryNetworkEntries []NetworkEntry `json:",omitempty"`
	Tags                    tags.Tags      `json:",omitempty"`
}

type MoveIpAddressesRequest struct {
	HypervisorHostname string
	IpAddresses        []net.IP
}

type MoveIpAddressesResponse struct {
	Error string
}

type NetworkEntry struct {
	Hostname       string       `json:",omitempty"`
	HostIpAddress  net.IP       `json:",omitempty"`
	HostMacAddress HardwareAddr `json:",omitempty"`
}
<|endoftext|>


<commit_before>package redisync

import (
	"testing"
	"time"
)

func TestLock(t *testing.T) {
	ttl := time.Second
	m, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	m.Lock()
	time.Sleep(ttl)
	ok := m.TryLock()
	if !ok {
		t.Error("Expected mutex to be lockable.")
		t.FailNow()
	}
	m.Unlock()
}

func TestLockLocked(t *testing.T) {
	ttl := time.Second
	m1, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	if ok := m1.TryLock(); !ok {
		t.Error("Expected mutex to be lockable.")
		t.FailNow()
	}
	m2, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	if ok := m2.TryLock(); ok {
		t.Error("Expected mutex not to be lockable.")
		t.FailNow()
	}
	m1.Unlock()
}

func TestUnlockOtherLocked(t *testing.T) {
	ttl := time.Second
	m1, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	if ok := m1.TryLock(); !ok {
		t.Error("Expected mutex to be lockable.")
		t.FailNow()
	}

	m2, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	if ok, _ := m2.Unlock(); ok {
		t.Error("Expected mutex not to be unlockable.")
		t.FailNow()
	}
}
<commit_msg>more tests<commit_after>package redisync

import (
	"testing"
	"time"
)

func TestLock(t *testing.T) {
	ttl := time.Second
	m, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	m.Lock()
	time.Sleep(ttl)
	ok := m.TryLock()
	if !ok {
		t.Error("Expected mutex to be lockable.")
		t.FailNow()
	}
	m.Unlock()
}

func TestLockLocked(t *testing.T) {
	ttl := time.Second
	m1, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	if ok := m1.TryLock(); !ok {
		t.Error("Expected mutex to be lockable.")
		t.FailNow()
	}
	m2, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	if ok := m2.TryLock(); ok {
		t.Error("Expected mutex not to be lockable.")
		t.FailNow()
	}
	m1.Unlock()
}

func TestUnlockOtherLocked(t *testing.T) {
	ttl := time.Second
	m1, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	if ok := m1.TryLock(); !ok {
		t.Error("Expected mutex to be lockable.")
		t.FailNow()
	}

	m2, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	if ok, _ := m2.Unlock(); ok {
		t.Error("Expected mutex not to be unlockable.")
		t.FailNow()
	}
	m1.Unlock()
}

func TestLockExpired(t *testing.T) {
	ttl := time.Second

	m1, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	if ok := m1.TryLock(); !ok {
		t.Error("Expected mutex to be lockable.")
		t.FailNow()
	}
	time.Sleep(ttl)

	m2, err := NewMutex("redisync.test.1", ttl, "")
	if err != nil {
		t.Error(err)
	}
	if ok := m2.TryLock(); !ok {
		t.Error("Expected mutex to be lockable.")
		t.FailNow()
	}
	m2.Unlock()
}
<|endoftext|>


<commit_before>// 24 may 2014
package main

import (
//	"fmt"
	"os"
	"strings"
	"go/token"
	"go/ast"
	"go/parser"
)

func main() {
	var pkg *ast.Package

	importpath := os.Args[1]

	fileset := token.NewFileSet()

	filter := func(i os.FileInfo) bool {
		return strings.HasSuffix(i.Name(), "_windows.go")
	}
	pkgs, err := parser.ParseDir(fileset, importpath,
		filter, parser.AllErrors)
	if err != nil {
		panic(err)
	}
	if len(pkgs) != 1 {
		panic("more than one package found")
	}
	for k, _ := range pkgs {		// get the sole key
		pkg = pkgs[k]
	}

	do(pkg)
	if len(unknown) > 0 {
		s := "error: the following are still unknown!"
		for k, _ := range unknown {
			s += "\n" + k
		}
		panic(s)
	}
}

type walker struct {
	desired	func(string) bool
}

var known = map[string]string{}
var unknown = map[string]struct{}{}

func (w *walker) Visit(node ast.Node) ast.Visitor {
	if n, ok := node.(*ast.Ident); ok {
		if w.desired(n.Name) {
			if n.Obj != nil {
				delete(unknown, n.Name)
				kind := n.Obj.Kind.String()
				if known[n.Name] != "" && known[n.Name] != kind {
					panic(n.Name + "(" + kind + ") already known to be a " + known[n.Name])
				}
				known[n.Name] = kind
			} else if _, ok := known[n.Name]; !ok {		// only if not known
				unknown[n.Name] = struct{}{}
			}
		}
	}
	return w
}

func do(pkg *ast.Package) {
	desired := func(name string) bool {
		if strings.HasPrefix(name, "_") && len(name) > 1 {
			return !strings.ContainsAny(name,
				"abcdefghijklmnopqrstuvwxyz")
		}
		return false
	}
	for _, f := range pkg.Files {
		for _, d := range f.Decls {
			ast.Walk(&walker{desired}, d)
		}
	}
}
<commit_msg>Refactored the constant gathering program a bit.<commit_after>// 24 may 2014
package main

import (
	"fmt"
	"os"
	"strings"
	"go/token"
	"go/ast"
	"go/parser"
)

func getPackage(path string) (pkg *ast.Package) {
	fileset := token.NewFileSet()		// parser.ParseDir() actually writes to this; not sure why it doesn't return one instead
	filter := func(i os.FileInfo) bool {
		return strings.HasSuffix(i.Name(), "_windows.go")
	}
	pkgs, err := parser.ParseDir(fileset, path, filter, parser.AllErrors)
	if err != nil {
		panic(err)
	}
	if len(pkgs) != 1 {
		panic("more than one package found")
	}
	for k, _ := range pkgs {		// get the sole key
		pkg = pkgs[k]
	}
	return pkg
}

type walker struct {
	desired	func(string) bool
}

var known = map[string]string{}
var unknown = map[string]struct{}{}

func (w *walker) Visit(node ast.Node) ast.Visitor {
	if n, ok := node.(*ast.Ident); ok {
		if w.desired(n.Name) {
			if n.Obj != nil {
				delete(unknown, n.Name)
				kind := n.Obj.Kind.String()
				if known[n.Name] != "" && known[n.Name] != kind {
					panic(n.Name + "(" + kind + ") already known to be a " + known[n.Name])
				}
				known[n.Name] = kind
			} else if _, ok := known[n.Name]; !ok {		// only if not known
				unknown[n.Name] = struct{}{}
			}
		}
	}
	return w
}

func gatherNames(pkg *ast.Package) {
	desired := func(name string) bool {
		if strings.HasPrefix(name, "_") && len(name) > 1 {
			return !strings.ContainsAny(name,
				"abcdefghijklmnopqrstuvwxyz")
		}
		return false
	}
	for _, f := range pkg.Files {
		for _, d := range f.Decls {
			ast.Walk(&walker{desired}, d)
		}
	}
}

func main() {
	if len(os.Args) != 2 {
		panic("usage: " + os.Args[0] + " path")
	}

	pkg := getPackage(os.Args[1])
	gatherNames(pkg)

	if len(unknown) > 0 {
		s := "error: the following are still unknown!"
		for k, _ := range unknown {
			s += "\n" + k
		}
		panic(s)
	}

	for ident, kind := range known {
		fmt.Printf("%-30s %s\n", ident, kind)
	}
}
<|endoftext|>


<commit_before>//
// Written by Michael Mattioli
//

package main

import (
    "fmt"
    "strconv"
    "os"
)

// Spiral is a square, 2D slice containing the numbers within the range of a particular number
// squared organized in a spiral fashion.
type Spiral struct {
    grid [][]int
}

// NewSpiral returns a pointer to a Spiral made with the specified number.
func NewSpiral(n int) *Spiral {

    // Make a new square, 2D grid.
    grid := make([][]int, n)
    for i := range grid {
        grid[i] = make([]int, n)
    }

    // Define the initial amount of steps to take when walking the grid in a spiral fashion.
    steps := map[string]int {
        "Right" : n - 1,
        "Down" : n - 2,
        "Left": n - 2,
        "Up" : n - 3,
    }

    // Starting point will be the top-left corner (0,0).
    position := struct {
        x, y int
    }{
        0,
        0,
    }

    // Always start to fill the grid with 1.
    fill := 1

    // Start by moving toward the right then down, then left, then up, etc.
    direction := "Right"

    // Keep track of the number of steps we've taken in each direction.
    distance := 0

    // We've reached the end of the line, turn.
    turn := func(d string) {
        steps[direction] -= 2
        distance = 0
        direction = d
    }

    // Keep walking down the line.
    walk := func() bool {
        if distance == steps[direction] {
            return false
        }
        distance++
        return true
    }

    for fill <= (n * n) { // While we haven't reached the final number...

        grid[position.x][position.y] = fill

        switch direction {
        case "Right":
            if walk() {
                position.x++
                break
            }
            position.y++
            turn("Down")
        case "Down":
            if walk() {
                position.y++
                break
            }
            position.x--
            turn("Left")
        case "Left":
            if walk() {
                position.x--
                break
            }
            position.y--
            turn("Up")
        case "Up":
            if walk() {
                position.y--
                break
            }
            position.x++
            turn("Right")
        }

        fill++

    }

    return &Spiral{grid}

}

func (s *Spiral) String() string {

    temp := ""

    width := len(strconv.Itoa(len(s.grid) * len(s.grid)))

    for i := range s.grid {
        for j := range s.grid[i] {
            element := strconv.Itoa(s.grid[j][i])
            for padding := len(element); padding <= width; padding++ {
                element += " "
            }
            temp += element
        }
        if i != len(s.grid) - 1 {
            temp += "\n"
        }
    }

    return temp

}

func main() {

    specimen, err := strconv.Atoi(os.Args[1])
    if err != nil {
        fmt.Println("Incorrect usage")
        os.Exit(1)
    }

    spiral := NewSpiral(specimen)
    fmt.Println(spiral)

}
<commit_msg>Improve string concatenation in challenge 320<commit_after>//
// Written by Michael Mattioli
//

package main

import (
    "fmt"
    "strconv"
    "bytes"
    "os"
)

// Spiral is a square, 2D slice containing the numbers within the range of a particular number
// squared organized in a spiral fashion.
type Spiral struct {
    grid [][]int
}

// NewSpiral returns a pointer to a Spiral made with the specified number.
func NewSpiral(n int) *Spiral {

    // Make a new square, 2D grid.
    grid := make([][]int, n)
    for i := range grid {
        grid[i] = make([]int, n)
    }

    // Define the initial amount of steps to take when walking the grid in a spiral fashion.
    steps := map[string]int {
        "Right" : n - 1,
        "Down" : n - 2,
        "Left": n - 2,
        "Up" : n - 3,
    }

    // Starting point will be the top-left corner (0,0).
    position := struct {
        x, y int
    }{
        0,
        0,
    }

    // Always start to fill the grid with 1.
    fill := 1

    // Start by moving toward the right then down, then left, then up, etc.
    direction := "Right"

    // Keep track of the number of steps we've taken in each direction.
    distance := 0

    // We've reached the end of the line, turn.
    turn := func(d string) {
        steps[direction] -= 2
        distance = 0
        direction = d
    }

    // Keep walking down the line.
    walk := func() bool {
        if distance == steps[direction] {
            return false
        }
        distance++
        return true
    }

    for fill <= (n * n) { // While we haven't reached the final number...

        grid[position.x][position.y] = fill

        switch direction {
        case "Right":
            if walk() {
                position.x++
                break
            }
            position.y++
            turn("Down")
        case "Down":
            if walk() {
                position.y++
                break
            }
            position.x--
            turn("Left")
        case "Left":
            if walk() {
                position.x--
                break
            }
            position.y--
            turn("Up")
        case "Up":
            if walk() {
                position.y--
                break
            }
            position.x++
            turn("Right")
        }

        fill++

    }

    return &Spiral{grid}

}

func (s *Spiral) String() string {

    var ns bytes.Buffer

    w := len(strconv.Itoa(len(s.grid) * len(s.grid)))

    for i := range s.grid {
        for j := range s.grid[i] {
            e := bytes.NewBufferString(strconv.Itoa(s.grid[j][i]))
            for p := len(e.String()); p <= w; p++ {
                e.WriteString(" ")
            }
            ns.WriteString(e.String())
        }
        if i != len(s.grid) - 1 {
            ns.WriteString("\n")
        }
    }

    return ns.String()

}

func main() {

    specimen, err := strconv.Atoi(os.Args[1])
    if err != nil {
        fmt.Println("Incorrect usage")
        os.Exit(1)
    }

    spiral := NewSpiral(specimen)
    fmt.Println(spiral)

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"

	"github.com/wellington/wellington"
)

// Sometimes circleci detects races in these tests. This may prevent it
var wtCmdMu sync.RWMutex

func init() {
	s := new(string)
	wtCmdMu.Lock()
	wtCmd.PersistentFlags().StringVarP(s, "test", "t", "", "dummy for testing")
	wtCmdMu.Unlock()

}

func resetFlags() {
	wtCmdMu.Lock()
	defer wtCmdMu.Unlock()
	wtCmd.ResetFlags()
}

func TestHTTP(t *testing.T) {
	wtCmd.SetArgs([]string{
		"serve",
	})

	// No way to shut this down
	go main()

	req, err := http.NewRequest("POST", "http://localhost:12345",
		bytes.NewBufferString(`div { p { color: red; } }`))

	if err != nil {
		t.Fatal(err)
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		t.Fatal(err)
	}
	if resp.Body == nil {
		t.Fatal("no response")
	}
	bs, _ := ioutil.ReadAll(resp.Body)

	var r wellington.Response
	err = json.Unmarshal(bs, &r)
	if err != nil {
		t.Fatal(err)
	}

	e := "/* line 1, stdin */\ndiv p {\n  color: red; }\n"
	if e != r.Contents {
		t.Errorf("got:\n%s\nwanted:\n%s", r.Contents, e)
	}
}

func TestStdin_import(t *testing.T) {
	resetFlags()

	oldOut := os.Stdout

	r, w, _ := os.Pipe()
	os.Stdout = w
	pwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}

	includeDir := filepath.Join(pwd, "..", "test", "sass")
	wtCmd.SetArgs([]string{
		"-p", includeDir,
		"compile", "../test/sass/import.scss"})
	main()

	outC := make(chan string)

	go func() {
		var buf bytes.Buffer
		io.Copy(&buf, r)
		outC <- buf.String()
	}()

	w.Close()
	os.Stdout = oldOut

	out := <-outC
	out = strings.Replace(out, includeDir, "", 1)
	e := `div {
  background: #00FF00;
  font-size: 10pt; }
`

	if !bytes.Contains([]byte(out), []byte(e)) {
		t.Errorf("got:\n%s\nwanted:\n%s", out, e)
	}

}

func TestStdin_sprite(t *testing.T) {
	resetFlags()

	oldStd := os.Stdin
	var oldOut *os.File
	oldOut = os.Stdout

	r, w, _ := os.Pipe()
	os.Stdout = w
	wtCmd.SetArgs([]string{
		"--dir", "../test/img",
		"--gen", "../test/img/build",
		"compile"})

	var err error
	os.Stdin, err = os.Open("../test/sass/sprite.scss")
	if err != nil {
		t.Fatal(err)
	}
	main()

	outC := make(chan string)

	go func() {
		var buf bytes.Buffer
		io.Copy(&buf, r)
		outC <- buf.String()
	}()

	w.Close()
	os.Stdin = oldStd
	os.Stdout = oldOut

	out := <-outC

	e := `div {
  height: 139px;
  width: 96px;
  background: url("../test/img/build/f0a220.png") 0px 0px; }
`

	if !bytes.Contains([]byte(out), []byte(e)) {
		t.Errorf("got:\n%s\nwanted:\n%s", out, e)
	}

}

func TestFile(t *testing.T) {
	// TODO: Tests for file importing here
}

func TestFile_comprehensive(t *testing.T) {
	resetFlags()

	oldStd := os.Stdin
	oldOut := os.Stdout

	r, w, _ := os.Pipe()
	os.Stdout = w
	wtCmd.SetArgs([]string{
		"--dir", "../test/img",
		"--gen", "../test/img/build",
		"--comment=false",
		"compile", "../test/comprehensive/compreh.scss"})
	main()

	outC := make(chan bytes.Buffer)

	go func() {
		var buf bytes.Buffer
		io.Copy(&buf, r)
		outC <- buf
	}()

	w.Close()
	os.Stdin = oldStd
	os.Stdout = oldOut

	out := <-outC

	e, err := ioutil.ReadFile("../test/comprehensive/expected.css")
	if err != nil {
		t.Fatal(err)
	}

	if bytes.Compare(out.Bytes(), e) != 0 {
		t.Errorf("got:\n%s\nwanted:\n%s", out.String(), string(e))
	}

}

func TestWatch_comprehensive(t *testing.T) {
	os.RemoveAll("../test/build/testwatch")
	resetFlags()

	wtCmd.SetArgs([]string{
		"--dir", "../test/img",
		"-b", "../test/build/testwatch",
		"--gen", "../test/build/testwatch/img",
		"--comment=false",
		"watch", "../test/comprehensive/compreh.scss",
	})
	main()
	_, err := os.Stat("../test/build/testwatch/compreh.css")
	if err != nil {
		t.Fatal(err)
	}

	_, err = os.Stat("../test/build/testwatch/img/5905b8.png")
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>maybe this is the source of data races<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"

	"github.com/wellington/wellington"
)

// Sometimes circleci detects races in these tests. This may prevent it
var wtCmdMu sync.RWMutex

func init() {
	s := new(string)
	wtCmdMu.Lock()
	wtCmd.PersistentFlags().StringVarP(s, "test", "t", "", "dummy for testing")
	wtCmdMu.Unlock()

}

func resetFlags() {
	wtCmdMu.Lock()
	defer wtCmdMu.Unlock()
	wtCmd.ResetFlags()
}

func TestHTTP(t *testing.T) {
	wtCmd.SetArgs([]string{
		"serve",
	})

	// No way to shut this down
	go func() {
		wtCmdMu.RLock()
		defer wtCmdMu.RUnlock()
		main()
	}()

	req, err := http.NewRequest("POST", "http://localhost:12345",
		bytes.NewBufferString(`div { p { color: red; } }`))

	if err != nil {
		t.Fatal(err)
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		t.Fatal(err)
	}
	if resp.Body == nil {
		t.Fatal("no response")
	}
	bs, _ := ioutil.ReadAll(resp.Body)

	var r wellington.Response
	err = json.Unmarshal(bs, &r)
	if err != nil {
		t.Fatal(err)
	}

	e := "/* line 1, stdin */\ndiv p {\n  color: red; }\n"
	if e != r.Contents {
		t.Errorf("got:\n%s\nwanted:\n%s", r.Contents, e)
	}
}

func TestStdin_import(t *testing.T) {
	resetFlags()

	oldOut := os.Stdout

	r, w, _ := os.Pipe()
	os.Stdout = w
	pwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}

	includeDir := filepath.Join(pwd, "..", "test", "sass")
	wtCmd.SetArgs([]string{
		"-p", includeDir,
		"compile", "../test/sass/import.scss"})
	main()

	outC := make(chan string)

	go func() {
		var buf bytes.Buffer
		io.Copy(&buf, r)
		outC <- buf.String()
	}()

	w.Close()
	os.Stdout = oldOut

	out := <-outC
	out = strings.Replace(out, includeDir, "", 1)
	e := `div {
  background: #00FF00;
  font-size: 10pt; }
`

	if !bytes.Contains([]byte(out), []byte(e)) {
		t.Errorf("got:\n%s\nwanted:\n%s", out, e)
	}

}

func TestStdin_sprite(t *testing.T) {
	resetFlags()

	oldStd := os.Stdin
	var oldOut *os.File
	oldOut = os.Stdout

	r, w, _ := os.Pipe()
	os.Stdout = w
	wtCmd.SetArgs([]string{
		"--dir", "../test/img",
		"--gen", "../test/img/build",
		"compile"})

	var err error
	os.Stdin, err = os.Open("../test/sass/sprite.scss")
	if err != nil {
		t.Fatal(err)
	}
	main()

	outC := make(chan string)

	go func() {
		var buf bytes.Buffer
		io.Copy(&buf, r)
		outC <- buf.String()
	}()

	w.Close()
	os.Stdin = oldStd
	os.Stdout = oldOut

	out := <-outC

	e := `div {
  height: 139px;
  width: 96px;
  background: url("../test/img/build/f0a220.png") 0px 0px; }
`

	if !bytes.Contains([]byte(out), []byte(e)) {
		t.Errorf("got:\n%s\nwanted:\n%s", out, e)
	}

}

func TestFile(t *testing.T) {
	// TODO: Tests for file importing here
}

func TestFile_comprehensive(t *testing.T) {
	resetFlags()

	oldStd := os.Stdin
	oldOut := os.Stdout

	r, w, _ := os.Pipe()
	os.Stdout = w
	wtCmd.SetArgs([]string{
		"--dir", "../test/img",
		"--gen", "../test/img/build",
		"--comment=false",
		"compile", "../test/comprehensive/compreh.scss"})
	main()

	outC := make(chan bytes.Buffer)

	go func() {
		var buf bytes.Buffer
		io.Copy(&buf, r)
		outC <- buf
	}()

	w.Close()
	os.Stdin = oldStd
	os.Stdout = oldOut

	out := <-outC

	e, err := ioutil.ReadFile("../test/comprehensive/expected.css")
	if err != nil {
		t.Fatal(err)
	}

	if bytes.Compare(out.Bytes(), e) != 0 {
		t.Errorf("got:\n%s\nwanted:\n%s", out.String(), string(e))
	}

}

func TestWatch_comprehensive(t *testing.T) {
	os.RemoveAll("../test/build/testwatch")
	resetFlags()

	wtCmd.SetArgs([]string{
		"--dir", "../test/img",
		"-b", "../test/build/testwatch",
		"--gen", "../test/build/testwatch/img",
		"--comment=false",
		"watch", "../test/comprehensive/compreh.scss",
	})
	main()
	_, err := os.Stat("../test/build/testwatch/compreh.css")
	if err != nil {
		t.Fatal(err)
	}

	_, err = os.Stat("../test/build/testwatch/img/5905b8.png")
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Drone.IO Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package webhook

import (
	"context"

	"github.com/drone/drone-go/drone"
)

// V1 is version 1 of the admission API
const V1 = "application/vnd.drone.webhook.v1+json"

// Webhook event types.
const (
	EventBuild = "build"
	EventRepo  = "repo"
	EventUser  = "user"
)

// Webhook action types.
const (
	ActionCreated  = "created"
	ActionUpdated  = "updated"
	ActionDeleted  = "deleted"
	ActionEnabled  = "enabled"
	ActionDisabled = "disabled"
)

type (
	// Request defines a webhook request.
	Request struct {
		Event  string       `json:"event"`
		Action string       `json:"action"`
		User   *drone.User  `json:"user,omitempty"`
		Repo   *drone.Repo  `json:"repo,omitempty"`
		Build  *drone.Build `json:"build,omitempty"`
	}

	// Plugin responds to a webhook request.
	Plugin interface {
		Deliver(context.Context, *Request) error
	}
)
<commit_msg>webhook: add missing field Request.System<commit_after>// Copyright 2018 Drone.IO Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package webhook

import (
	"context"

	"github.com/drone/drone-go/drone"
)

// V1 is version 1 of the admission API
const V1 = "application/vnd.drone.webhook.v1+json"

// Webhook event types.
const (
	EventBuild = "build"
	EventRepo  = "repo"
	EventUser  = "user"
)

// Webhook action types.
const (
	ActionCreated  = "created"
	ActionUpdated  = "updated"
	ActionDeleted  = "deleted"
	ActionEnabled  = "enabled"
	ActionDisabled = "disabled"
)

type (
	// Request defines a webhook request.
	Request struct {
		Event  string        `json:"event"`
		Action string        `json:"action"`
		User   *drone.User   `json:"user,omitempty"`
		Repo   *drone.Repo   `json:"repo,omitempty"`
		Build  *drone.Build  `json:"build,omitempty"`
		System *drone.System `json:"system,omitempty"`
	}

	// Plugin responds to a webhook request.
	Plugin interface {
		Deliver(context.Context, *Request) error
	}
)
<|endoftext|>