<commit_before>package restpc

import (
	"fmt"
	"time"
)

var FromContext FromX = &fromContext{}

type fromContext struct{}

func (f *fromContext) GetString(req Request, key string) (*string, error) {
	ctx := req.HTTP().Context()
	valueIn := ctx.Value(key)
	if valueIn == nil {
		return nil, nil
	}
	switch value := valueIn.(type) {
	case string:
		valueStr := value // to copy
		return &valueStr, nil
	case []byte:
		valueStr := string(value)
		return &valueStr, nil
	default:
		return nil, NewError(
			InvalidArgument,
			fmt.Sprintf("invalid '%v', must be string", key),
			fmt.Errorf("ctx.Value(%#v) = %#v", key, valueIn),
			"ctx", ctx,
		)
	}
	return nil, nil
}
func (f *fromContext) GetStringList(req Request, key string) ([]string, error) {
	ctx := req.HTTP().Context()
	valueIn := ctx.Value(key)
	if valueIn == nil {
		return nil, nil
	}
	switch value := valueIn.(type) {
	case []string:
		valueSlice := append([]string(nil), value...) // to copy
		return valueSlice, nil
	default:
		return nil, NewError(
			InvalidArgument,
			fmt.Sprintf("invalid '%v', must be string", key),
			fmt.Errorf("ctx.Value(%#v) = %#v", key, valueIn),
			"ctx", ctx,
		)
	}
	return nil, nil
}
func (f *fromContext) GetInt(req Request, key string) (*int, error) {
	ctx := req.HTTP().Context()
	valueIn := ctx.Value(key)
	if valueIn == nil {
		return nil, nil
	}
	switch value := valueIn.(type) {
	case float64:
		valueInt := int(value)
		return &valueInt, nil
	case int:
		valueInt := value // to copy
		return &valueInt, nil
	case int32:
		valueInt := int(value)
		return &valueInt, nil
	case int64:
		valueInt := int(value)
		return &valueInt, nil
	default:
		return nil, NewError(
			InvalidArgument,
			fmt.Sprintf("invalid '%v', must be string", key),
			fmt.Errorf("ctx.Value(%#v) = %#v", key, valueIn),
			"ctx", ctx,
		)
	}
	return nil, nil
}
func (f *fromContext) GetFloat(req Request, key string) (*float64, error) {
	ctx := req.HTTP().Context()
	valueIn := ctx.Value(key)
	if valueIn == nil {
		return nil, nil
	}
	switch value := valueIn.(type) {
	case float64:
		valueF := value // to copy
		return &valueF, nil
	case float32:
		valueF := float64(value)
		return &valueF, nil
	case int:
		valueF := float64(value)
		return &valueF, nil
	case int64:
		valueF := float64(value)
		return &valueF, nil
	case int32:
		valueF := float64(value)
		return &valueF, nil
	default:
		return nil, NewError(
			InvalidArgument,
			fmt.Sprintf("invalid '%v', must be string", key),
			fmt.Errorf("ctx.Value(%#v) = %#v", key, valueIn),
			"ctx", ctx,
		)
	}
	return nil, nil
}
func (f *fromContext) GetBool(req Request, key string) (*bool, error) {
	ctx := req.HTTP().Context()
	valueIn := ctx.Value(key)
	if valueIn == nil {
		return nil, nil
	}
	switch value := valueIn.(type) {
	case bool:
		valueBool := value // to copy
		return &valueBool, nil
	default:
		return nil, NewError(
			InvalidArgument,
			fmt.Sprintf("invalid '%v', must be string", key),
			fmt.Errorf("ctx.Value(%#v) = %#v", key, valueIn),
			"ctx", ctx,
		)
	}
	return nil, nil
}
func (f *fromContext) GetTime(req Request, key string) (*time.Time, error) {
	ctx := req.HTTP().Context()
	valueIn := ctx.Value(key)
	if valueIn == nil {
		return nil, nil
	}
	switch value := valueIn.(type) {
	case time.Time:
		valueTime := value // to copy
		return &valueTime, nil
	case string:
		valueTm, err := time.Parse(time.RFC3339, value)
		if err != nil {
			return nil, NewError(
				InvalidArgument,
				fmt.Sprintf("invalid '%v', must be RFC3339 time string", key),
				err,
				"value", value,
			)
		}
		return &valueTm, nil
	default:
		return nil, NewError(
			InvalidArgument,
			fmt.Sprintf("invalid '%v', must be string", key),
			fmt.Errorf("ctx.Value(%#v) = %#v", key, valueIn),
			"ctx", ctx,
		)
	}
	return nil, nil
}
<commit_msg>fix error message in fromContext.GetStringList<commit_after>package restpc

import (
	"fmt"
	"time"
)

var FromContext FromX = &fromContext{}

type fromContext struct{}

func (f *fromContext) GetString(req Request, key string) (*string, error) {
	ctx := req.HTTP().Context()
	valueIn := ctx.Value(key)
	if valueIn == nil {
		return nil, nil
	}
	switch value := valueIn.(type) {
	case string:
		valueStr := value // to copy
		return &valueStr, nil
	case []byte:
		valueStr := string(value)
		return &valueStr, nil
	default:
		return nil, NewError(
			InvalidArgument,
			fmt.Sprintf("invalid '%v', must be string", key),
			fmt.Errorf("ctx.Value(%#v) = %#v", key, valueIn),
			"ctx", ctx,
		)
	}
	return nil, nil
}
func (f *fromContext) GetStringList(req Request, key string) ([]string, error) {
	ctx := req.HTTP().Context()
	valueIn := ctx.Value(key)
	if valueIn == nil {
		return nil, nil
	}
	switch value := valueIn.(type) {
	case []string:
		valueSlice := append([]string(nil), value...) // to copy
		return valueSlice, nil
	default:
		return nil, NewError(
			InvalidArgument,
			fmt.Sprintf("invalid '%v', must be array of strings", key),
			fmt.Errorf("ctx.Value(%#v) = %#v", key, valueIn),
			"ctx", ctx,
		)
	}
	return nil, nil
}
func (f *fromContext) GetInt(req Request, key string) (*int, error) {
	ctx := req.HTTP().Context()
	valueIn := ctx.Value(key)
	if valueIn == nil {
		return nil, nil
	}
	switch value := valueIn.(type) {
	case float64:
		valueInt := int(value)
		return &valueInt, nil
	case int:
		valueInt := value // to copy
		return &valueInt, nil
	case int32:
		valueInt := int(value)
		return &valueInt, nil
	case int64:
		valueInt := int(value)
		return &valueInt, nil
	default:
		return nil, NewError(
			InvalidArgument,
			fmt.Sprintf("invalid '%v', must be string", key),
			fmt.Errorf("ctx.Value(%#v) = %#v", key, valueIn),
			"ctx", ctx,
		)
	}
	return nil, nil
}
func (f *fromContext) GetFloat(req Request, key string) (*float64, error) {
	ctx := req.HTTP().Context()
	valueIn := ctx.Value(key)
	if valueIn == nil {
		return nil, nil
	}
	switch value := valueIn.(type) {
	case float64:
		valueF := value // to copy
		return &valueF, nil
	case float32:
		valueF := float64(value)
		return &valueF, nil
	case int:
		valueF := float64(value)
		return &valueF, nil
	case int64:
		valueF := float64(value)
		return &valueF, nil
	case int32:
		valueF := float64(value)
		return &valueF, nil
	default:
		return nil, NewError(
			InvalidArgument,
			fmt.Sprintf("invalid '%v', must be string", key),
			fmt.Errorf("ctx.Value(%#v) = %#v", key, valueIn),
			"ctx", ctx,
		)
	}
	return nil, nil
}
func (f *fromContext) GetBool(req Request, key string) (*bool, error) {
	ctx := req.HTTP().Context()
	valueIn := ctx.Value(key)
	if valueIn == nil {
		return nil, nil
	}
	switch value := valueIn.(type) {
	case bool:
		valueBool := value // to copy
		return &valueBool, nil
	default:
		return nil, NewError(
			InvalidArgument,
			fmt.Sprintf("invalid '%v', must be string", key),
			fmt.Errorf("ctx.Value(%#v) = %#v", key, valueIn),
			"ctx", ctx,
		)
	}
	return nil, nil
}
func (f *fromContext) GetTime(req Request, key string) (*time.Time, error) {
	ctx := req.HTTP().Context()
	valueIn := ctx.Value(key)
	if valueIn == nil {
		return nil, nil
	}
	switch value := valueIn.(type) {
	case time.Time:
		valueTime := value // to copy
		return &valueTime, nil
	case string:
		valueTm, err := time.Parse(time.RFC3339, value)
		if err != nil {
			return nil, NewError(
				InvalidArgument,
				fmt.Sprintf("invalid '%v', must be RFC3339 time string", key),
				err,
				"value", value,
			)
		}
		return &valueTm, nil
	default:
		return nil, NewError(
			InvalidArgument,
			fmt.Sprintf("invalid '%v', must be string", key),
			fmt.Errorf("ctx.Value(%#v) = %#v", key, valueIn),
			"ctx", ctx,
		)
	}
	return nil, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cert

import (
	"bytes"
	"crypto"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	cryptorand "crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"errors"
	"fmt"
	"io/ioutil"
	"math"
	"math/big"
	"net"
	"path"
	"strings"
	"time"
)

const (
	rsaKeySize   = 2048
	duration365d = time.Hour * 24 * 365
)

// Config contains the basic fields required for creating a certificate
type Config struct {
	CommonName   string
	Organization []string
	AltNames     AltNames
	Usages       []x509.ExtKeyUsage
}

// AltNames contains the domain names and IP addresses that will be added
// to the API Server's x509 certificate SubAltNames field. The values will
// be passed directly to the x509.Certificate object.
type AltNames struct {
	DNSNames []string
	IPs      []net.IP
}

// NewPrivateKey creates an RSA private key
func NewPrivateKey() (*rsa.PrivateKey, error) {
	return rsa.GenerateKey(cryptorand.Reader, rsaKeySize)
}

// NewSelfSignedCACert creates a CA certificate
func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, error) {
	now := time.Now()
	tmpl := x509.Certificate{
		SerialNumber: new(big.Int).SetInt64(0),
		Subject: pkix.Name{
			CommonName:   cfg.CommonName,
			Organization: cfg.Organization,
		},
		NotBefore:             now.UTC(),
		NotAfter:              now.Add(duration365d * 10).UTC(),
		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
		BasicConstraintsValid: true,
		IsCA:                  true,
	}

	certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &tmpl, &tmpl, key.Public(), key)
	if err != nil {
		return nil, err
	}
	return x509.ParseCertificate(certDERBytes)
}

// NewSignedCert creates a signed certificate using the given CA certificate and key
func NewSignedCert(cfg Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) {
	serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64))
	if err != nil {
		return nil, err
	}
	if len(cfg.CommonName) == 0 {
		return nil, errors.New("must specify a CommonName")
	}
	if len(cfg.Usages) == 0 {
		return nil, errors.New("must specify at least one ExtKeyUsage")
	}

	certTmpl := x509.Certificate{
		Subject: pkix.Name{
			CommonName:   cfg.CommonName,
			Organization: cfg.Organization,
		},
		DNSNames:     cfg.AltNames.DNSNames,
		IPAddresses:  cfg.AltNames.IPs,
		SerialNumber: serial,
		NotBefore:    caCert.NotBefore,
		NotAfter:     time.Now().Add(duration365d).UTC(),
		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:  cfg.Usages,
	}
	certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey)
	if err != nil {
		return nil, err
	}
	return x509.ParseCertificate(certDERBytes)
}

// MakeEllipticPrivateKeyPEM creates an ECDSA private key
func MakeEllipticPrivateKeyPEM() ([]byte, error) {
	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
	if err != nil {
		return nil, err
	}

	derBytes, err := x509.MarshalECPrivateKey(privateKey)
	if err != nil {
		return nil, err
	}

	privateKeyPemBlock := &pem.Block{
		Type:  ECPrivateKeyBlockType,
		Bytes: derBytes,
	}
	return pem.EncodeToMemory(privateKeyPemBlock), nil
}

// GenerateSelfSignedCertKey creates a self-signed certificate and key for the given host.
// Host may be an IP or a DNS name
// You may also specify additional subject alt names (either ip or dns names) for the certificate.
func GenerateSelfSignedCertKey(host string, alternateIPs []net.IP, alternateDNS []string) ([]byte, []byte, error) {
	return GenerateSelfSignedCertKeyWithFixtures(host, alternateIPs, alternateDNS, "")
}

// GenerateSelfSignedCertKeyWithFixtures creates a self-signed certificate and key for the given host.
// Host may be an IP or a DNS name. You may also specify additional subject alt names (either ip or dns names)
// for the certificate.
//
// If fixtureDirectory is non-empty, it is a directory path which can contain pre-generated certs. The format is:
// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.crt
// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.key
// Certs/keys not existing in that directory are created.
func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, alternateDNS []string, fixtureDirectory string) ([]byte, []byte, error) {
	validFrom := time.Now().Add(-time.Hour) // valid an hour earlier to avoid flakes due to clock skew
	maxAge := time.Hour * 24 * 365          // one year self-signed certs

	baseName := fmt.Sprintf("%s_%s_%s", host, strings.Join(ipsToStrings(alternateIPs), "-"), strings.Join(alternateDNS, "-"))
	certFixturePath := path.Join(fixtureDirectory, baseName+".crt")
	keyFixturePath := path.Join(fixtureDirectory, baseName+".key")
	if len(fixtureDirectory) > 0 {
		cert, err := ioutil.ReadFile(certFixturePath)
		if err == nil {
			key, err := ioutil.ReadFile(keyFixturePath)
			if err == nil {
				return cert, key, nil
			}
			return nil, nil, fmt.Errorf("cert %s can be read, but key %s cannot: %v", certFixturePath, keyFixturePath, err)
		}
		maxAge = 100 * time.Hour * 24 * 365 // 100 years fixtures
	}

	caKey, err := rsa.GenerateKey(cryptorand.Reader, 2048)
	if err != nil {
		return nil, nil, err
	}

	caTemplate := x509.Certificate{
		SerialNumber: big.NewInt(1),
		Subject: pkix.Name{
			CommonName: fmt.Sprintf("%s-ca@%d", host, time.Now().Unix()),
		},
		NotBefore: validFrom,
		NotAfter:  validFrom.Add(maxAge),

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
		BasicConstraintsValid: true,
		IsCA:                  true,
	}

	caDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &caTemplate, &caTemplate, &caKey.PublicKey, caKey)
	if err != nil {
		return nil, nil, err
	}

	caCertificate, err := x509.ParseCertificate(caDERBytes)
	if err != nil {
		return nil, nil, err
	}

	priv, err := rsa.GenerateKey(cryptorand.Reader, 2048)
	if err != nil {
		return nil, nil, err
	}

	template := x509.Certificate{
		SerialNumber: big.NewInt(2),
		Subject: pkix.Name{
			CommonName: fmt.Sprintf("%s@%d", host, time.Now().Unix()),
		},
		NotBefore: validFrom,
		NotAfter:  validFrom.Add(maxAge),

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
	}

	if ip := net.ParseIP(host); ip != nil {
		template.IPAddresses = append(template.IPAddresses, ip)
	} else {
		template.DNSNames = append(template.DNSNames, host)
	}

	template.IPAddresses = append(template.IPAddresses, alternateIPs...)
	template.DNSNames = append(template.DNSNames, alternateDNS...)

	derBytes, err := x509.CreateCertificate(cryptorand.Reader, &template, caCertificate, &priv.PublicKey, caKey)
	if err != nil {
		return nil, nil, err
	}

	// Generate cert, followed by ca
	certBuffer := bytes.Buffer{}
	if err := pem.Encode(&certBuffer, &pem.Block{Type: CertificateBlockType, Bytes: derBytes}); err != nil {
		return nil, nil, err
	}
	if err := pem.Encode(&certBuffer, &pem.Block{Type: CertificateBlockType, Bytes: caDERBytes}); err != nil {
		return nil, nil, err
	}

	// Generate key
	keyBuffer := bytes.Buffer{}
	if err := pem.Encode(&keyBuffer, &pem.Block{Type: RSAPrivateKeyBlockType, Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
		return nil, nil, err
	}

	if len(fixtureDirectory) > 0 {
		if err := ioutil.WriteFile(certFixturePath, certBuffer.Bytes(), 0644); err != nil {
			return nil, nil, fmt.Errorf("failed to write cert fixture to %s: %v", certFixturePath, err)
		}
		if err := ioutil.WriteFile(keyFixturePath, keyBuffer.Bytes(), 0644); err != nil {
			return nil, nil, fmt.Errorf("failed to write key fixture to %s: %v", certFixturePath, err)
		}
	}

	return certBuffer.Bytes(), keyBuffer.Bytes(), nil
}

// FormatBytesCert receives byte array certificate and formats in human-readable format
func FormatBytesCert(cert []byte) (string, error) {
	block, _ := pem.Decode(cert)
	c, err := x509.ParseCertificate(block.Bytes)
	if err != nil {
		return "", fmt.Errorf("failed to parse certificate [%v]", err)
	}
	return FormatCert(c), nil
}

// FormatCert receives certificate and formats in human-readable format
func FormatCert(c *x509.Certificate) string {
	var ips []string
	for _, ip := range c.IPAddresses {
		ips = append(ips, ip.String())
	}
	altNames := append(ips, c.DNSNames...)
	res := fmt.Sprintf(
		"Issuer: CN=%s | Subject: CN=%s | CA: %t\n",
		c.Issuer.CommonName, c.Subject.CommonName, c.IsCA,
	)
	res += fmt.Sprintf("Not before: %s Not After: %s", c.NotBefore, c.NotAfter)
	if len(altNames) > 0 {
		res += fmt.Sprintf("\nAlternate Names: %v", altNames)
	}
	return res
}

func ipsToStrings(ips []net.IP) []string {
	ss := make([]string, 0, len(ips))
	for _, ip := range ips {
		ss = append(ss, ip.String())
	}
	return ss
}
<commit_msg>Remove unused funcs in certutil<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cert

import (
	"bytes"
	"crypto"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	cryptorand "crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"errors"
	"fmt"
	"io/ioutil"
	"math"
	"math/big"
	"net"
	"path"
	"strings"
	"time"
)

const (
	rsaKeySize   = 2048
	duration365d = time.Hour * 24 * 365
)

// Config contains the basic fields required for creating a certificate
type Config struct {
	CommonName   string
	Organization []string
	AltNames     AltNames
	Usages       []x509.ExtKeyUsage
}

// AltNames contains the domain names and IP addresses that will be added
// to the API Server's x509 certificate SubAltNames field. The values will
// be passed directly to the x509.Certificate object.
type AltNames struct {
	DNSNames []string
	IPs      []net.IP
}

// NewPrivateKey creates an RSA private key
func NewPrivateKey() (*rsa.PrivateKey, error) {
	return rsa.GenerateKey(cryptorand.Reader, rsaKeySize)
}

// NewSelfSignedCACert creates a CA certificate
func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, error) {
	now := time.Now()
	tmpl := x509.Certificate{
		SerialNumber: new(big.Int).SetInt64(0),
		Subject: pkix.Name{
			CommonName:   cfg.CommonName,
			Organization: cfg.Organization,
		},
		NotBefore:             now.UTC(),
		NotAfter:              now.Add(duration365d * 10).UTC(),
		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
		BasicConstraintsValid: true,
		IsCA:                  true,
	}

	certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &tmpl, &tmpl, key.Public(), key)
	if err != nil {
		return nil, err
	}
	return x509.ParseCertificate(certDERBytes)
}

// NewSignedCert creates a signed certificate using the given CA certificate and key
func NewSignedCert(cfg Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) {
	serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64))
	if err != nil {
		return nil, err
	}
	if len(cfg.CommonName) == 0 {
		return nil, errors.New("must specify a CommonName")
	}
	if len(cfg.Usages) == 0 {
		return nil, errors.New("must specify at least one ExtKeyUsage")
	}

	certTmpl := x509.Certificate{
		Subject: pkix.Name{
			CommonName:   cfg.CommonName,
			Organization: cfg.Organization,
		},
		DNSNames:     cfg.AltNames.DNSNames,
		IPAddresses:  cfg.AltNames.IPs,
		SerialNumber: serial,
		NotBefore:    caCert.NotBefore,
		NotAfter:     time.Now().Add(duration365d).UTC(),
		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:  cfg.Usages,
	}
	certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey)
	if err != nil {
		return nil, err
	}
	return x509.ParseCertificate(certDERBytes)
}

// MakeEllipticPrivateKeyPEM creates an ECDSA private key
func MakeEllipticPrivateKeyPEM() ([]byte, error) {
	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
	if err != nil {
		return nil, err
	}

	derBytes, err := x509.MarshalECPrivateKey(privateKey)
	if err != nil {
		return nil, err
	}

	privateKeyPemBlock := &pem.Block{
		Type:  ECPrivateKeyBlockType,
		Bytes: derBytes,
	}
	return pem.EncodeToMemory(privateKeyPemBlock), nil
}

// GenerateSelfSignedCertKey creates a self-signed certificate and key for the given host.
// Host may be an IP or a DNS name
// You may also specify additional subject alt names (either ip or dns names) for the certificate.
func GenerateSelfSignedCertKey(host string, alternateIPs []net.IP, alternateDNS []string) ([]byte, []byte, error) {
	return GenerateSelfSignedCertKeyWithFixtures(host, alternateIPs, alternateDNS, "")
}

// GenerateSelfSignedCertKeyWithFixtures creates a self-signed certificate and key for the given host.
// Host may be an IP or a DNS name. You may also specify additional subject alt names (either ip or dns names)
// for the certificate.
//
// If fixtureDirectory is non-empty, it is a directory path which can contain pre-generated certs. The format is:
// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.crt
// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.key
// Certs/keys not existing in that directory are created.
func GenerateSelfSignedCertKeyWithFixtures(host string, alternateIPs []net.IP, alternateDNS []string, fixtureDirectory string) ([]byte, []byte, error) {
	validFrom := time.Now().Add(-time.Hour) // valid an hour earlier to avoid flakes due to clock skew
	maxAge := time.Hour * 24 * 365          // one year self-signed certs

	baseName := fmt.Sprintf("%s_%s_%s", host, strings.Join(ipsToStrings(alternateIPs), "-"), strings.Join(alternateDNS, "-"))
	certFixturePath := path.Join(fixtureDirectory, baseName+".crt")
	keyFixturePath := path.Join(fixtureDirectory, baseName+".key")
	if len(fixtureDirectory) > 0 {
		cert, err := ioutil.ReadFile(certFixturePath)
		if err == nil {
			key, err := ioutil.ReadFile(keyFixturePath)
			if err == nil {
				return cert, key, nil
			}
			return nil, nil, fmt.Errorf("cert %s can be read, but key %s cannot: %v", certFixturePath, keyFixturePath, err)
		}
		maxAge = 100 * time.Hour * 24 * 365 // 100 years fixtures
	}

	caKey, err := rsa.GenerateKey(cryptorand.Reader, 2048)
	if err != nil {
		return nil, nil, err
	}

	caTemplate := x509.Certificate{
		SerialNumber: big.NewInt(1),
		Subject: pkix.Name{
			CommonName: fmt.Sprintf("%s-ca@%d", host, time.Now().Unix()),
		},
		NotBefore: validFrom,
		NotAfter:  validFrom.Add(maxAge),

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
		BasicConstraintsValid: true,
		IsCA:                  true,
	}

	caDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &caTemplate, &caTemplate, &caKey.PublicKey, caKey)
	if err != nil {
		return nil, nil, err
	}

	caCertificate, err := x509.ParseCertificate(caDERBytes)
	if err != nil {
		return nil, nil, err
	}

	priv, err := rsa.GenerateKey(cryptorand.Reader, 2048)
	if err != nil {
		return nil, nil, err
	}

	template := x509.Certificate{
		SerialNumber: big.NewInt(2),
		Subject: pkix.Name{
			CommonName: fmt.Sprintf("%s@%d", host, time.Now().Unix()),
		},
		NotBefore: validFrom,
		NotAfter:  validFrom.Add(maxAge),

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
	}

	if ip := net.ParseIP(host); ip != nil {
		template.IPAddresses = append(template.IPAddresses, ip)
	} else {
		template.DNSNames = append(template.DNSNames, host)
	}

	template.IPAddresses = append(template.IPAddresses, alternateIPs...)
	template.DNSNames = append(template.DNSNames, alternateDNS...)

	derBytes, err := x509.CreateCertificate(cryptorand.Reader, &template, caCertificate, &priv.PublicKey, caKey)
	if err != nil {
		return nil, nil, err
	}

	// Generate cert, followed by ca
	certBuffer := bytes.Buffer{}
	if err := pem.Encode(&certBuffer, &pem.Block{Type: CertificateBlockType, Bytes: derBytes}); err != nil {
		return nil, nil, err
	}
	if err := pem.Encode(&certBuffer, &pem.Block{Type: CertificateBlockType, Bytes: caDERBytes}); err != nil {
		return nil, nil, err
	}

	// Generate key
	keyBuffer := bytes.Buffer{}
	if err := pem.Encode(&keyBuffer, &pem.Block{Type: RSAPrivateKeyBlockType, Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
		return nil, nil, err
	}

	if len(fixtureDirectory) > 0 {
		if err := ioutil.WriteFile(certFixturePath, certBuffer.Bytes(), 0644); err != nil {
			return nil, nil, fmt.Errorf("failed to write cert fixture to %s: %v", certFixturePath, err)
		}
		if err := ioutil.WriteFile(keyFixturePath, keyBuffer.Bytes(), 0644); err != nil {
			return nil, nil, fmt.Errorf("failed to write key fixture to %s: %v", certFixturePath, err)
		}
	}

	return certBuffer.Bytes(), keyBuffer.Bytes(), nil
}

func ipsToStrings(ips []net.IP) []string {
	ss := make([]string, 0, len(ips))
	for _, ip := range ips {
		ss = append(ss, ip.String())
	}
	return ss
}
<|endoftext|>


<commit_before>package util

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/Sirupsen/logrus"
	"github.com/stretchr/testify/assert"
)

func init() {
	AppPath = filepath.Clean(AppPath + "/../" + "test")
}

func removeTestDirectory() {
	err := os.RemoveAll(AppPath)

	if err != nil {
		logrus.Fatal(err)
	}
}

func createTestDirectory() {
	err := os.Mkdir(AppPath, 0777)

	if err != nil && !os.IsExist(err) {
		logrus.Fatal(err)
	}
}

func createSubTestDirectory(path string) {
	err := os.Mkdir(fmt.Sprintf("%s%c%s", AppPath, filepath.Separator, path), 0777)

	if err != nil && !os.IsExist(err) {
		logrus.Fatal(err)
	}
}

func TestGetAbsPath(t *testing.T) {
	AppPath = "/tmp"

	assert.Equal(t, "/tmp/file", GetAbsPath("file"), "Must return an absolute path")
}

func TestTrimAbsBasePath(t *testing.T) {
	AppPath = "/tmp"

	assert.Equal(t, "test", TrimAbsBasePath("/tmp/test"), "Must return relative path")
}

func TestGetFolderPathAddTrailingSeparator(t *testing.T) {
	assert.Equal(t, "test/", GetFolderPath("test"), "Must add a trailing seperator")
}

func TestGetFolderPathAddTrailingSeparatorWithMultipleTrailingSeparator(t *testing.T) {
	assert.Equal(t, "test/", GetFolderPath("test////////////"), "Must add a trailing seperator")
}

func TestExtractFolderFiles(t *testing.T) {
	removeTestDirectory()
	createTestDirectory()
	createSubTestDirectory("test1")
	createSubTestDirectory("test1/test2")
	createSubTestDirectory("test1/test2/test3")

	for _, filename := range []string{"file1", "test1/file2", "test1/test2/file3", "test1/test2/test3/file4", "test1/test2/test3/file5"} {
		content := []byte("This is a file")
		err := ioutil.WriteFile(GetAbsPath(filename), content, 0644)

		if err != nil {
			logrus.Fatal(err)
		}
	}

	assert.Equal(t, &[]string{"file1", "test1/file2", "test1/test2/file3", "test1/test2/test3/file4", "test1/test2/test3/file5"}, ExtractFolderFiles("/"), "Must extract all files in folder")
}
<commit_msg>test(util) : fix<commit_after>package util

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/Sirupsen/logrus"
	"github.com/stretchr/testify/assert"
)

func init() {
	AppPath = filepath.Clean(AppPath + "/../" + "test")
}

func removeTestDirectory() {
	err := os.RemoveAll(AppPath)

	if err != nil {
		logrus.Fatal(err)
	}
}

func createTestDirectory() {
	err := os.Mkdir(AppPath, 0777)

	if err != nil && !os.IsExist(err) {
		logrus.Fatal(err)
	}
}

func createSubTestDirectory(path string) {
	err := os.Mkdir(fmt.Sprintf("%s%c%s", AppPath, filepath.Separator, path), 0777)

	if err != nil && !os.IsExist(err) {
		logrus.Fatal(err)
	}
}

func TestGetAbsPath(t *testing.T) {
	assert.Equal(t, AppPath+"/file", GetAbsPath("file"), "Must return an absolute path")
}

func TestTrimAbsBasePath(t *testing.T) {
	assert.Equal(t, "test", TrimAbsBasePath(AppPath+"/test"), "Must return relative path")
}

func TestGetFolderPathAddTrailingSeparator(t *testing.T) {
	assert.Equal(t, "test/", GetFolderPath("test"), "Must add a trailing seperator")
}

func TestGetFolderPathAddTrailingSeparatorWithMultipleTrailingSeparator(t *testing.T) {
	assert.Equal(t, "test/", GetFolderPath("test////////////"), "Must add a trailing seperator")
}

func TestExtractFolderFiles(t *testing.T) {
	removeTestDirectory()
	createTestDirectory()
	createSubTestDirectory("test1")
	createSubTestDirectory("test1/test2")
	createSubTestDirectory("test1/test2/test3")

	for _, filename := range []string{"file1", "test1/file2", "test1/test2/file3", "test1/test2/test3/file4", "test1/test2/test3/file5"} {
		content := []byte("This is a file")
		err := ioutil.WriteFile(GetAbsPath(filename), content, 0644)

		if err != nil {
			logrus.Fatal(err)
		}
	}

	assert.Equal(t, &[]string{"file1", "test1/file2", "test1/test2/file3", "test1/test2/test3/file4", "test1/test2/test3/file5"}, ExtractFolderFiles("/"), "Must extract all files in folder")
}
<|endoftext|>


<commit_before>package main

func main() {
	// ...
}
<commit_msg>Add check command. Refs #1742<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/af83/edwig/siri"
	"time"
)

func main() {
	flag.Parse()

	command := flag.Args()[0]

	var err error
	switch command {
	case "check":
		err = checkStatus(flag.Args()[1])
	}

	if err != nil {
		panic(err)
	}
}

func checkStatus(url string) error {
	client := siri.NewSOAPClient(url)
	request := &siri.SIRICheckStatusRequest{
		RequestorRef:      "NINOXE:default",
		RequestTimestamp:  time.Now(), // FIXME Use clock. See #1735
		MessageIdentifier: "CheckStatus:Test:0",
	}
	response, err := client.CheckStatus(request)
	if err != nil {
		return err
	}
	fmt.Println(response)
	return nil
}
<|endoftext|>


<commit_before>package inigo_test

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"

	"github.com/cloudfoundry-incubator/inigo/loggredile"
	"github.com/fraenkel/candiedyaml"

	"github.com/cloudfoundry-incubator/inigo/inigo_server"
	"github.com/cloudfoundry-incubator/runtime-schema/models/factories"
	"github.com/cloudfoundry-incubator/stager/integration/stager_runner"
	"github.com/cloudfoundry/yagnats"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	zip_helper "github.com/pivotal-golang/archiver/extractor/test_helper"
)

func downloadBuildArtifactsCache(appId string) []byte {
	fileServerUrl := fmt.Sprintf("http://127.0.0.1:%d/v1/build_artifacts/%s", suiteContext.FileServerPort, appId)
	resp, err := http.Get(fileServerUrl)
	Ω(err).ShouldNot(HaveOccurred())

	Ω(resp.StatusCode).Should(Equal(http.StatusOK))

	bytes, err := ioutil.ReadAll(resp.Body)
	Ω(err).ShouldNot(HaveOccurred())

	return bytes
}

var _ = Describe("Stager", func() {
	var otherStagerRunner *stager_runner.StagerRunner
	var appId string
	var taskId string

	BeforeEach(func() {
		appId = factories.GenerateGuid()
		taskId = factories.GenerateGuid()
		suiteContext.FileServerRunner.Start()
		otherStagerRunner = stager_runner.New(
			suiteContext.SharedContext.StagerPath,
			suiteContext.EtcdRunner.NodeURLS(),
			[]string{fmt.Sprintf("127.0.0.1:%d", suiteContext.NatsPort)},
		)
	})

	Context("when unable to find an appropriate compiler", func() {
		BeforeEach(func() {
			suiteContext.ExecutorRunner.Start()
			suiteContext.RepRunner.Start()
			suiteContext.StagerRunner.Start(
				"--diskMB", "128",
				"--memoryMB", "128",
			)
		})

		It("returns an error", func() {
			receivedMessages := make(chan *yagnats.Message)
			suiteContext.NatsRunner.MessageBus.Subscribe("diego.staging.finished", func(message *yagnats.Message) {
				receivedMessages <- message
			})

			suiteContext.NatsRunner.MessageBus.Publish(
				"diego.staging.start",
				[]byte(fmt.Sprintf(`{
					"app_id": "%s",
					"task_id": "%s",
					"app_bits_download_uri": "some-download-uri",
					"build_artifacts_cache_download_uri": "artifacts-download-uri",
					"build_artifacts_cache_upload_uri": "artifacts-upload-uri",
					"stack": "no-circus"
				}`, appId, taskId)),
			)

			var receivedMessage *yagnats.Message
			Eventually(receivedMessages, SHORT_TIMEOUT).Should(Receive(&receivedMessage))
			Ω(receivedMessage.Payload).Should(ContainSubstring("no compiler defined for requested stack"))
			Consistently(receivedMessages, SHORT_TIMEOUT).ShouldNot(Receive())
		})
	})

	Describe("Staging", func() {
		var outputGuid string
		var stagingMessage []byte
		var buildpackToUse string

		BeforeEach(func() {
			suiteContext.ExecutorRunner.Start()
			suiteContext.RepRunner.Start()

			buildpackToUse = "admin_buildpack.zip"
			outputGuid = factories.GenerateGuid()

			suiteContext.FileServerRunner.ServeFile("lifecycle.zip", suiteContext.SharedContext.CircusZipPath)

			//make and upload an app
			var appFiles = []zip_helper.ArchiveFile{
				{Name: "my-app", Body: "scooby-doo"},
			}

			zip_helper.CreateZipArchive("/tmp/app.zip", appFiles)
			inigo_server.UploadFile("app.zip", "/tmp/app.zip")

			//make and upload a buildpack
			var adminBuildpackFiles = []zip_helper.ArchiveFile{
				{
					Name: "bin/detect",
					Body: `#!/bin/bash
echo My Buildpack
				`},
				{
					Name: "bin/compile",
					Body: `#!/bin/bash
echo $1 $2
echo COMPILING BUILDPACK
echo $SOME_STAGING_ENV
touch $1/compiled
touch $2/inserted-into-artifacts-cache
				`},
				{
					Name: "bin/release",
					Body: `#!/bin/bash
cat <<EOF
---
default_process_types:
  web: start-command
EOF
				`},
			}
			zip_helper.CreateZipArchive("/tmp/admin_buildpack.zip", adminBuildpackFiles)
			inigo_server.UploadFile("admin_buildpack.zip", "/tmp/admin_buildpack.zip")

			var bustedAdminBuildpackFiles = []zip_helper.ArchiveFile{
				{
					Name: "bin/detect",
					Body: `#!/bin/bash]
				exit 1
				`},
				{Name: "bin/compile", Body: `#!/bin/bash`},
				{Name: "bin/release", Body: `#!/bin/bash`},
			}

			zip_helper.CreateZipArchive("/tmp/busted_admin_buildpack.zip", bustedAdminBuildpackFiles)
			inigo_server.UploadFile("busted_admin_buildpack.zip", "/tmp/busted_admin_buildpack.zip")
		})

		JustBeforeEach(func() {
			stagingMessage = []byte(
				fmt.Sprintf(
					`{
						"app_id": "%s",
						"task_id": "%s",
						"memory_mb": 128,
						"disk_mb": 128,
						"file_descriptors": 1024,
						"stack": "lucid64",
						"app_bits_download_uri": "%s",
						"buildpacks" : [{ "name": "test-buildpack", "key": "test-buildpack-key", "url": "%s" }],
						"environment": [{ "name": "SOME_STAGING_ENV", "value": "%s"}]
					}`,
					appId,
					taskId,
					inigo_server.DownloadUrl("app.zip"),
					inigo_server.DownloadUrl(buildpackToUse),
					outputGuid,
				),
			)
		})

		Context("with one stager running", func() {
			BeforeEach(func() {
				suiteContext.StagerRunner.Start(
					"--circuses", `{"lucid64":"lifecycle.zip"}`,
					"--diskMB", "128",
					"--memoryMB", "128",
				)
			})

			It("runs the compiler on the executor with the correct environment variables, bits and log tag, and responds with the detected buildpack", func() {
				//listen for NATS response
				payloads := make(chan []byte)

				suiteContext.NatsRunner.MessageBus.Subscribe("diego.staging.finished", func(msg *yagnats.Message) {
					payloads <- msg.Payload
				})

				//stream logs
				messages, stop := loggredile.StreamMessages(
					suiteContext.LoggregatorRunner.Config.OutgoingPort,
					fmt.Sprintf("/tail/?app=%s", appId),
				)
				defer close(stop)

				logOutput := gbytes.NewBuffer()
				go func() {
					for message := range messages {
						Ω(message.GetSourceName()).To(Equal("STG"))
						logOutput.Write([]byte(string(message.GetMessage()) + "\n"))
					}
				}()

				//publish the staging message
				err := suiteContext.NatsRunner.MessageBus.Publish("diego.staging.start", stagingMessage)
				Ω(err).ShouldNot(HaveOccurred())

				//wait for staging to complete
				var payload []byte
				Eventually(payloads, LONG_TIMEOUT).Should(Receive(&payload))

				//Assert on the staging output (detected buildpack)
				Ω(string(payload)).Should(MatchJSON(fmt.Sprintf(`{
					"app_id": "%s",
					"task_id": "%s",
					"buildpack_key":"test-buildpack-key",
					"detected_buildpack":"My Buildpack",
					"detected_start_command":"start-command"
				}`, appId, taskId)))

				//Asser the user saw reasonable output
				Eventually(logOutput).Should(gbytes.Say("COMPILING BUILDPACK"))
				Ω(logOutput.Contents()).Should(ContainSubstring(outputGuid))

				// Assert that the build artifacts cache was downloaded
				//TODO: how do we test they were downloaded??

				// Download the build artifacts cache from the file-server
				buildArtifactsCacheBytes := downloadBuildArtifactsCache(appId)
				Ω(buildArtifactsCacheBytes).ShouldNot(BeEmpty())

				// Assert that the downloaded build artifacts cache matches what the buildpack created
				artifactsCache, err := gzip.NewReader(bytes.NewReader(buildArtifactsCacheBytes))
				Ω(err).ShouldNot(HaveOccurred())

				untarredBuildArtifactsData := tar.NewReader(artifactsCache)
				buildArtifactContents := map[string][]byte{}
				for {
					hdr, err := untarredBuildArtifactsData.Next()
					if err == io.EOF {
						break
					}

					Ω(err).ShouldNot(HaveOccurred())

					content, err := ioutil.ReadAll(untarredBuildArtifactsData)
					Ω(err).ShouldNot(HaveOccurred())

					buildArtifactContents[hdr.Name] = content
				}

				//Ω(buildArtifactContents).Should(HaveKey("pulled-down-from-artifacts-cache"))
				Ω(buildArtifactContents).Should(HaveKey("./inserted-into-artifacts-cache"))

				//Fetch the compiled droplet from the fakeCC
				dropletData, ok := suiteContext.FakeCC.UploadedDroplets[appId]
				Ω(ok).Should(BeTrue())
				Ω(dropletData).ShouldNot(BeEmpty())

				//Unzip the droplet
				ungzippedDropletData, err := gzip.NewReader(bytes.NewReader(dropletData))
				Ω(err).ShouldNot(HaveOccurred())

				//Untar the droplet
				untarredDropletData := tar.NewReader(ungzippedDropletData)
				dropletContents := map[string][]byte{}
				for {
					hdr, err := untarredDropletData.Next()
					if err == io.EOF {
						break
					}
					Ω(err).ShouldNot(HaveOccurred())

					content, err := ioutil.ReadAll(untarredDropletData)
					Ω(err).ShouldNot(HaveOccurred())

					dropletContents[hdr.Name] = content
				}

				//Assert the droplet has the right files in it
				Ω(dropletContents).Should(HaveKey("./"))
				Ω(dropletContents).Should(HaveKey("./staging_info.yml"))
				Ω(dropletContents).Should(HaveKey("./logs/"))
				Ω(dropletContents).Should(HaveKey("./tmp/"))
				Ω(dropletContents).Should(HaveKey("./app/"))
				Ω(dropletContents).Should(HaveKey("./app/my-app"))
				Ω(dropletContents).Should(HaveKey("./app/compiled"))

				//Assert the files contain the right content
				Ω(string(dropletContents["./app/my-app"])).Should(Equal("scooby-doo"))

				//In particular, staging_info.yml should have the correct detected_buildpack and start_command
				yamlDecoder := candiedyaml.NewDecoder(bytes.NewReader(dropletContents["./staging_info.yml"]))
				stagingInfo := map[string]string{}
				err = yamlDecoder.Decode(&stagingInfo)
				Ω(err).ShouldNot(HaveOccurred())

				Ω(stagingInfo["detected_buildpack"]).Should(Equal("My Buildpack"))
				Ω(stagingInfo["start_command"]).Should(Equal("start-command"))

				//Assert nothing else crept into the droplet
				Ω(dropletContents).Should(HaveLen(7))
			})

			Context("when compilation fails", func() {
				BeforeEach(func() {
					buildpackToUse = "busted_admin_buildpack.zip"
				})

				It("responds with the error, and no detected buildpack present", func() {
					payloads := make(chan []byte)

					suiteContext.NatsRunner.MessageBus.Subscribe("diego.staging.finished", func(msg *yagnats.Message) {
						payloads <- msg.Payload
					})

					messages, stop := loggredile.StreamMessages(
						suiteContext.LoggregatorRunner.Config.OutgoingPort,
						fmt.Sprintf("/tail/?app=%s", appId),
					)
					defer close(stop)

					logOutput := gbytes.NewBuffer()
					go func() {
						for message := range messages {
							Ω(message.GetSourceName()).To(Equal("STG"))
							logOutput.Write([]byte(string(message.GetMessage()) + "\n"))
						}
					}()

					err := suiteContext.NatsRunner.MessageBus.Publish(
						"diego.staging.start",
						stagingMessage,
					)
					Ω(err).ShouldNot(HaveOccurred())

					var payload []byte
					Eventually(payloads, 10.0).Should(Receive(&payload))
					Ω(string(payload)).Should(MatchJSON(fmt.Sprintf(`{
						"app_id":"%s",
						"task_id":"%s",
						"error":"Exited with status 1"
					}`, appId, taskId)))

					Eventually(logOutput).Should(gbytes.Say("no valid buildpacks detected"))
				})
			})
		})

		Context("with two stagers running", func() {
			BeforeEach(func() {
				suiteContext.StagerRunner.Start(
					"--circuses", `{"lucid64":"lifecycle.zip"}`,
					"--diskMB", "128",
					"--memoryMB", "128",
				)
				otherStagerRunner.Start("--circuses", `{"lucid64":"lifecycle.zip"}`)
			})

			AfterEach(func() {
				otherStagerRunner.Stop()
			})

			It("only one returns a staging completed response", func() {
				received := make(chan bool)

				_, err := suiteContext.NatsRunner.MessageBus.Subscribe("diego.staging.finished", func(message *yagnats.Message) {
					received <- true
				})
				Ω(err).ShouldNot(HaveOccurred())

				err = suiteContext.NatsRunner.MessageBus.Publish(
					"diego.staging.start",
					stagingMessage,
				)
				Ω(err).ShouldNot(HaveOccurred())

				Eventually(received, LONG_TIMEOUT).Should(Receive())
				Consistently(received, SHORT_TIMEOUT).ShouldNot(Receive())
			})
		})
	})
})
<commit_msg>Rename stager resource limit flags<commit_after>package inigo_test

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"

	"github.com/cloudfoundry-incubator/inigo/loggredile"
	"github.com/fraenkel/candiedyaml"

	"github.com/cloudfoundry-incubator/inigo/inigo_server"
	"github.com/cloudfoundry-incubator/runtime-schema/models/factories"
	"github.com/cloudfoundry-incubator/stager/integration/stager_runner"
	"github.com/cloudfoundry/yagnats"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	zip_helper "github.com/pivotal-golang/archiver/extractor/test_helper"
)

func downloadBuildArtifactsCache(appId string) []byte {
	fileServerUrl := fmt.Sprintf("http://127.0.0.1:%d/v1/build_artifacts/%s", suiteContext.FileServerPort, appId)
	resp, err := http.Get(fileServerUrl)
	Ω(err).ShouldNot(HaveOccurred())

	Ω(resp.StatusCode).Should(Equal(http.StatusOK))

	bytes, err := ioutil.ReadAll(resp.Body)
	Ω(err).ShouldNot(HaveOccurred())

	return bytes
}

var _ = Describe("Stager", func() {
	var otherStagerRunner *stager_runner.StagerRunner
	var appId string
	var taskId string

	BeforeEach(func() {
		appId = factories.GenerateGuid()
		taskId = factories.GenerateGuid()
		suiteContext.FileServerRunner.Start()
		otherStagerRunner = stager_runner.New(
			suiteContext.SharedContext.StagerPath,
			suiteContext.EtcdRunner.NodeURLS(),
			[]string{fmt.Sprintf("127.0.0.1:%d", suiteContext.NatsPort)},
		)
	})

	Context("when unable to find an appropriate compiler", func() {
		BeforeEach(func() {
			suiteContext.ExecutorRunner.Start()
			suiteContext.RepRunner.Start()
			suiteContext.StagerRunner.Start(
				"--minDiskMB", "64",
				"--minMemoryMB", "64",
			)
		})

		It("returns an error", func() {
			receivedMessages := make(chan *yagnats.Message)
			suiteContext.NatsRunner.MessageBus.Subscribe("diego.staging.finished", func(message *yagnats.Message) {
				receivedMessages <- message
			})

			suiteContext.NatsRunner.MessageBus.Publish(
				"diego.staging.start",
				[]byte(fmt.Sprintf(`{
					"app_id": "%s",
					"task_id": "%s",
					"app_bits_download_uri": "some-download-uri",
					"build_artifacts_cache_download_uri": "artifacts-download-uri",
					"build_artifacts_cache_upload_uri": "artifacts-upload-uri",
					"stack": "no-circus"
				}`, appId, taskId)),
			)

			var receivedMessage *yagnats.Message
			Eventually(receivedMessages, SHORT_TIMEOUT).Should(Receive(&receivedMessage))
			Ω(receivedMessage.Payload).Should(ContainSubstring("no compiler defined for requested stack"))
			Consistently(receivedMessages, SHORT_TIMEOUT).ShouldNot(Receive())
		})
	})

	Describe("Staging", func() {
		var outputGuid string
		var stagingMessage []byte
		var buildpackToUse string

		BeforeEach(func() {
			suiteContext.ExecutorRunner.Start()
			suiteContext.RepRunner.Start()

			buildpackToUse = "admin_buildpack.zip"
			outputGuid = factories.GenerateGuid()

			suiteContext.FileServerRunner.ServeFile("lifecycle.zip", suiteContext.SharedContext.CircusZipPath)

			//make and upload an app
			var appFiles = []zip_helper.ArchiveFile{
				{Name: "my-app", Body: "scooby-doo"},
			}

			zip_helper.CreateZipArchive("/tmp/app.zip", appFiles)
			inigo_server.UploadFile("app.zip", "/tmp/app.zip")

			//make and upload a buildpack
			var adminBuildpackFiles = []zip_helper.ArchiveFile{
				{
					Name: "bin/detect",
					Body: `#!/bin/bash
echo My Buildpack
				`},
				{
					Name: "bin/compile",
					Body: `#!/bin/bash
echo $1 $2
echo COMPILING BUILDPACK
echo $SOME_STAGING_ENV
touch $1/compiled
touch $2/inserted-into-artifacts-cache
				`},
				{
					Name: "bin/release",
					Body: `#!/bin/bash
cat <<EOF
---
default_process_types:
  web: start-command
EOF
				`},
			}
			zip_helper.CreateZipArchive("/tmp/admin_buildpack.zip", adminBuildpackFiles)
			inigo_server.UploadFile("admin_buildpack.zip", "/tmp/admin_buildpack.zip")

			var bustedAdminBuildpackFiles = []zip_helper.ArchiveFile{
				{
					Name: "bin/detect",
					Body: `#!/bin/bash]
				exit 1
				`},
				{Name: "bin/compile", Body: `#!/bin/bash`},
				{Name: "bin/release", Body: `#!/bin/bash`},
			}

			zip_helper.CreateZipArchive("/tmp/busted_admin_buildpack.zip", bustedAdminBuildpackFiles)
			inigo_server.UploadFile("busted_admin_buildpack.zip", "/tmp/busted_admin_buildpack.zip")
		})

		JustBeforeEach(func() {
			stagingMessage = []byte(
				fmt.Sprintf(
					`{
						"app_id": "%s",
						"task_id": "%s",
						"memory_mb": 128,
						"disk_mb": 128,
						"file_descriptors": 1024,
						"stack": "lucid64",
						"app_bits_download_uri": "%s",
						"buildpacks" : [{ "name": "test-buildpack", "key": "test-buildpack-key", "url": "%s" }],
						"environment": [{ "name": "SOME_STAGING_ENV", "value": "%s"}]
					}`,
					appId,
					taskId,
					inigo_server.DownloadUrl("app.zip"),
					inigo_server.DownloadUrl(buildpackToUse),
					outputGuid,
				),
			)
		})

		Context("with one stager running", func() {
			BeforeEach(func() {
				suiteContext.StagerRunner.Start(
					"--circuses", `{"lucid64":"lifecycle.zip"}`,
					"--minDiskMB", "64",
					"--minMemoryMB", "64",
				)
			})

			It("runs the compiler on the executor with the correct environment variables, bits and log tag, and responds with the detected buildpack", func() {
				//listen for NATS response
				payloads := make(chan []byte)

				suiteContext.NatsRunner.MessageBus.Subscribe("diego.staging.finished", func(msg *yagnats.Message) {
					payloads <- msg.Payload
				})

				//stream logs
				messages, stop := loggredile.StreamMessages(
					suiteContext.LoggregatorRunner.Config.OutgoingPort,
					fmt.Sprintf("/tail/?app=%s", appId),
				)
				defer close(stop)

				logOutput := gbytes.NewBuffer()
				go func() {
					for message := range messages {
						Ω(message.GetSourceName()).To(Equal("STG"))
						logOutput.Write([]byte(string(message.GetMessage()) + "\n"))
					}
				}()

				//publish the staging message
				err := suiteContext.NatsRunner.MessageBus.Publish("diego.staging.start", stagingMessage)
				Ω(err).ShouldNot(HaveOccurred())

				//wait for staging to complete
				var payload []byte
				Eventually(payloads, LONG_TIMEOUT).Should(Receive(&payload))

				//Assert on the staging output (detected buildpack)
				Ω(string(payload)).Should(MatchJSON(fmt.Sprintf(`{
					"app_id": "%s",
					"task_id": "%s",
					"buildpack_key":"test-buildpack-key",
					"detected_buildpack":"My Buildpack",
					"detected_start_command":"start-command"
				}`, appId, taskId)))

				//Asser the user saw reasonable output
				Eventually(logOutput).Should(gbytes.Say("COMPILING BUILDPACK"))
				Ω(logOutput.Contents()).Should(ContainSubstring(outputGuid))

				// Assert that the build artifacts cache was downloaded
				//TODO: how do we test they were downloaded??

				// Download the build artifacts cache from the file-server
				buildArtifactsCacheBytes := downloadBuildArtifactsCache(appId)
				Ω(buildArtifactsCacheBytes).ShouldNot(BeEmpty())

				// Assert that the downloaded build artifacts cache matches what the buildpack created
				artifactsCache, err := gzip.NewReader(bytes.NewReader(buildArtifactsCacheBytes))
				Ω(err).ShouldNot(HaveOccurred())

				untarredBuildArtifactsData := tar.NewReader(artifactsCache)
				buildArtifactContents := map[string][]byte{}
				for {
					hdr, err := untarredBuildArtifactsData.Next()
					if err == io.EOF {
						break
					}

					Ω(err).ShouldNot(HaveOccurred())

					content, err := ioutil.ReadAll(untarredBuildArtifactsData)
					Ω(err).ShouldNot(HaveOccurred())

					buildArtifactContents[hdr.Name] = content
				}

				//Ω(buildArtifactContents).Should(HaveKey("pulled-down-from-artifacts-cache"))
				Ω(buildArtifactContents).Should(HaveKey("./inserted-into-artifacts-cache"))

				//Fetch the compiled droplet from the fakeCC
				dropletData, ok := suiteContext.FakeCC.UploadedDroplets[appId]
				Ω(ok).Should(BeTrue())
				Ω(dropletData).ShouldNot(BeEmpty())

				//Unzip the droplet
				ungzippedDropletData, err := gzip.NewReader(bytes.NewReader(dropletData))
				Ω(err).ShouldNot(HaveOccurred())

				//Untar the droplet
				untarredDropletData := tar.NewReader(ungzippedDropletData)
				dropletContents := map[string][]byte{}
				for {
					hdr, err := untarredDropletData.Next()
					if err == io.EOF {
						break
					}
					Ω(err).ShouldNot(HaveOccurred())

					content, err := ioutil.ReadAll(untarredDropletData)
					Ω(err).ShouldNot(HaveOccurred())

					dropletContents[hdr.Name] = content
				}

				//Assert the droplet has the right files in it
				Ω(dropletContents).Should(HaveKey("./"))
				Ω(dropletContents).Should(HaveKey("./staging_info.yml"))
				Ω(dropletContents).Should(HaveKey("./logs/"))
				Ω(dropletContents).Should(HaveKey("./tmp/"))
				Ω(dropletContents).Should(HaveKey("./app/"))
				Ω(dropletContents).Should(HaveKey("./app/my-app"))
				Ω(dropletContents).Should(HaveKey("./app/compiled"))

				//Assert the files contain the right content
				Ω(string(dropletContents["./app/my-app"])).Should(Equal("scooby-doo"))

				//In particular, staging_info.yml should have the correct detected_buildpack and start_command
				yamlDecoder := candiedyaml.NewDecoder(bytes.NewReader(dropletContents["./staging_info.yml"]))
				stagingInfo := map[string]string{}
				err = yamlDecoder.Decode(&stagingInfo)
				Ω(err).ShouldNot(HaveOccurred())

				Ω(stagingInfo["detected_buildpack"]).Should(Equal("My Buildpack"))
				Ω(stagingInfo["start_command"]).Should(Equal("start-command"))

				//Assert nothing else crept into the droplet
				Ω(dropletContents).Should(HaveLen(7))
			})

			Context("when compilation fails", func() {
				BeforeEach(func() {
					buildpackToUse = "busted_admin_buildpack.zip"
				})

				It("responds with the error, and no detected buildpack present", func() {
					payloads := make(chan []byte)

					suiteContext.NatsRunner.MessageBus.Subscribe("diego.staging.finished", func(msg *yagnats.Message) {
						payloads <- msg.Payload
					})

					messages, stop := loggredile.StreamMessages(
						suiteContext.LoggregatorRunner.Config.OutgoingPort,
						fmt.Sprintf("/tail/?app=%s", appId),
					)
					defer close(stop)

					logOutput := gbytes.NewBuffer()
					go func() {
						for message := range messages {
							Ω(message.GetSourceName()).To(Equal("STG"))
							logOutput.Write([]byte(string(message.GetMessage()) + "\n"))
						}
					}()

					err := suiteContext.NatsRunner.MessageBus.Publish(
						"diego.staging.start",
						stagingMessage,
					)
					Ω(err).ShouldNot(HaveOccurred())

					var payload []byte
					Eventually(payloads, 10.0).Should(Receive(&payload))
					Ω(string(payload)).Should(MatchJSON(fmt.Sprintf(`{
						"app_id":"%s",
						"task_id":"%s",
						"error":"Exited with status 1"
					}`, appId, taskId)))

					Eventually(logOutput).Should(gbytes.Say("no valid buildpacks detected"))
				})
			})
		})

		Context("with two stagers running", func() {
			BeforeEach(func() {
				suiteContext.StagerRunner.Start(
					"--circuses", `{"lucid64":"lifecycle.zip"}`,
					"--minDiskMB", "64",
					"--minMemoryMB", "64",
				)
				otherStagerRunner.Start(
					"--circuses", `{"lucid64":"lifecycle.zip"}`,
					"--minDiskMB", "64",
					"--minMemoryMB", "64",
				)
			})

			AfterEach(func() {
				otherStagerRunner.Stop()
			})

			It("only one returns a staging completed response", func() {
				received := make(chan bool)

				_, err := suiteContext.NatsRunner.MessageBus.Subscribe("diego.staging.finished", func(message *yagnats.Message) {
					received <- true
				})
				Ω(err).ShouldNot(HaveOccurred())

				err = suiteContext.NatsRunner.MessageBus.Publish(
					"diego.staging.start",
					stagingMessage,
				)
				Ω(err).ShouldNot(HaveOccurred())

				Eventually(received, LONG_TIMEOUT).Should(Receive())
				Consistently(received, SHORT_TIMEOUT).ShouldNot(Receive())
			})
		})
	})
})
<|endoftext|>


<commit_before>package godirwalk

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
)

var rootDir string

func TestMain(m *testing.M) {
	flag.Parse()

	// All tests use the same directory test scaffolding.  Create the directory
	// hierarchy, run the tests, then remove the root directory of the test
	// scaffolding.
	if err := setup(); err != nil {
		fmt.Fprintf(os.Stderr, "setup: %s\n", err)
		os.Exit(1)
	}

	code := m.Run()

	if err := teardown(); err != nil {
		fmt.Fprintf(os.Stderr, "teardown: %s\n", err)
		os.Exit(1)
	}

	os.Exit(code)
}

func setup() error {
	var err error

	rootDir, err = ioutil.TempDir(os.TempDir(), "godirwalk-")
	if err != nil {
		return err
	}

	// Create files, creating parent directories along the way.
	files := []string{
		"dir1/dir1a/file1a1",
		"dir1/dir1a/skip",
		"dir1/dir1a/z1a2",
		"dir1/file1b",
		"dir2/file2a",
		"dir2/skip/file2b1",
		"dir2/z2c/file2c1",
		"dir3/aaa.txt",
		"dir3/zzz/aaa.txt",
		"dir4/aaa.txt",
		"dir4/zzz/aaa.txt",
		"dir5/a1.txt",
		"dir5/a2/a2a/a2a1.txt",
		"dir5/a2/a2b.txt",
		"dir6/bravo.txt",
		"dir6/code/123.txt",
		"dir7/z",
		"file3",
	}

	for _, pathname := range files {
		pathname = filepath.Join(rootDir, filepath.FromSlash(pathname))

		if err := os.MkdirAll(filepath.Dir(pathname), os.ModePerm); err != nil {
			return fmt.Errorf("cannot create directory for test scaffolding: %s", err)
		}

		if err = ioutil.WriteFile(pathname, []byte("some test data\n"), os.ModePerm); err != nil {
			return fmt.Errorf("cannot create file for test scaffolding: %s", err)
		}
	}

	symlinks := []struct {
		newname, oldname string
	}{
		{"dir3/skip", "zzz"},
		{"dir4/symlinkToDirectory", "zzz"},
		{"dir4/symlinkToFile", "aaa.txt"},
		{"dir7/a/x", "../b"},
		{"dir7/b/y", "../z"},
		{"symlinks/dir-symlink", "../symlinks"}, // infinite loop of symlinks
		{"symlinks/file-symlink", "../file3"},
		{"symlinks/invalid-symlink", "/non/existing/file"},
	}

	for _, entry := range symlinks {
		newname := filepath.Join(rootDir, filepath.FromSlash(entry.newname))

		if err := os.MkdirAll(filepath.Dir(newname), os.ModePerm); err != nil {
			return fmt.Errorf("cannot create directory for test scaffolding: %s", err)
		}

		oldname := filepath.FromSlash(entry.oldname)

		if err := os.Symlink(oldname, newname); err != nil {
			return fmt.Errorf("cannot create symbolic link for test scaffolding: %s", err)
		}
	}

	extraDirs := []string{
		"dir6/abc",
		"dir6/def",
	}

	for _, pathname := range extraDirs {
		pathname = filepath.Join(rootDir, filepath.FromSlash(pathname))

		if err := os.MkdirAll(pathname, os.ModePerm); err != nil {
			return fmt.Errorf("cannot create directory for test scaffolding: %s", err)
		}
	}

	if err := os.MkdirAll(filepath.Join(rootDir, filepath.FromSlash("dir6/noaccess")), 0 /* no permissions */); err != nil {
		return fmt.Errorf("cannot create directory for test scaffolding: %s", err)
	}

	return nil
}

func teardown() error {
	if err := os.Chmod(filepath.Join(rootDir, filepath.FromSlash("dir6/noaccess")), os.ModePerm); err != nil {
		return fmt.Errorf("cannot change permission to delete dir6/noaccess for test scaffolding: %s", err)
	}
	if err := os.RemoveAll(rootDir); err != nil {
		return err
	}
	return nil
}
<commit_msg>scaffolding validates noaccess file mode<commit_after>package godirwalk

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
)

var rootDir string

func TestMain(m *testing.M) {
	flag.Parse()

	// All tests use the same directory test scaffolding.  Create the directory
	// hierarchy, run the tests, then remove the root directory of the test
	// scaffolding.
	if err := setup(); err != nil {
		fmt.Fprintf(os.Stderr, "setup: %s\n", err)
		os.Exit(1)
	}

	code := m.Run()

	if err := teardown(); err != nil {
		fmt.Fprintf(os.Stderr, "teardown: %s\n", err)
		os.Exit(1)
	}

	os.Exit(code)
}

func setup() error {
	var err error

	rootDir, err = ioutil.TempDir(os.TempDir(), "godirwalk-")
	if err != nil {
		return err
	}

	// Create files, creating parent directories along the way.
	files := []string{
		"dir1/dir1a/file1a1",
		"dir1/dir1a/skip",
		"dir1/dir1a/z1a2",
		"dir1/file1b",
		"dir2/file2a",
		"dir2/skip/file2b1",
		"dir2/z2c/file2c1",
		"dir3/aaa.txt",
		"dir3/zzz/aaa.txt",
		"dir4/aaa.txt",
		"dir4/zzz/aaa.txt",
		"dir5/a1.txt",
		"dir5/a2/a2a/a2a1.txt",
		"dir5/a2/a2b.txt",
		"dir6/bravo.txt",
		"dir6/code/123.txt",
		"dir7/z",
		"file3",
	}

	for _, pathname := range files {
		pathname = filepath.Join(rootDir, filepath.FromSlash(pathname))

		if err := os.MkdirAll(filepath.Dir(pathname), os.ModePerm); err != nil {
			return fmt.Errorf("cannot create directory for test scaffolding: %s", err)
		}

		if err = ioutil.WriteFile(pathname, []byte("some test data\n"), os.ModePerm); err != nil {
			return fmt.Errorf("cannot create file for test scaffolding: %s", err)
		}
	}

	symlinks := []struct {
		newname, oldname string
	}{
		{"dir3/skip", "zzz"},
		{"dir4/symlinkToDirectory", "zzz"},
		{"dir4/symlinkToFile", "aaa.txt"},
		{"dir7/a/x", "../b"},
		{"dir7/b/y", "../z"},
		{"symlinks/dir-symlink", "../symlinks"}, // infinite loop of symlinks
		{"symlinks/file-symlink", "../file3"},
		{"symlinks/invalid-symlink", "/non/existing/file"},
	}

	for _, entry := range symlinks {
		newname := filepath.Join(rootDir, filepath.FromSlash(entry.newname))

		if err := os.MkdirAll(filepath.Dir(newname), os.ModePerm); err != nil {
			return fmt.Errorf("cannot create directory for test scaffolding: %s", err)
		}

		oldname := filepath.FromSlash(entry.oldname)

		if err := os.Symlink(oldname, newname); err != nil {
			return fmt.Errorf("cannot create symbolic link for test scaffolding: %s", err)
		}
	}

	extraDirs := []string{
		"dir6/abc",
		"dir6/def",
	}

	for _, pathname := range extraDirs {
		pathname = filepath.Join(rootDir, filepath.FromSlash(pathname))

		if err := os.MkdirAll(pathname, os.ModePerm); err != nil {
			return fmt.Errorf("cannot create directory for test scaffolding: %s", err)
		}
	}

	fn := filepath.Join(rootDir, filepath.FromSlash("dir6/noaccess"))
	fm := os.FileMode(0)
	if err := os.MkdirAll(fn, fm); err != nil {
		return fmt.Errorf("cannot create directory for test scaffolding: %s", err)
	}
	fi, err := os.Lstat(fn)
	if err != nil {
		return fmt.Errorf("cannot stat for test scaffolding: %s", err)
	}
	if got, want := fi.Mode()&os.ModePerm, fm; got != want {
		return fmt.Errorf("%s: GOT: %v; WANT: %v", fn, got, want)
	}

	return nil
}

func teardown() error {
	if err := os.Chmod(filepath.Join(rootDir, filepath.FromSlash("dir6/noaccess")), os.ModePerm); err != nil {
		return fmt.Errorf("cannot change permission to delete dir6/noaccess for test scaffolding: %s", err)
	}
	if err := os.RemoveAll(rootDir); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package util

import (
	"fmt"
	"io"
	"strings"
)

type Traceback struct {
	Name   string
	Source string
	Begin  int
	End    int
	Next   *Traceback
}

var CulpritStyle = "1;4"

func (te *Traceback) Pprint(w io.Writer, sourceIndent string) {
	if te.Begin == -1 {
		fmt.Fprintf(w, "%s, unknown position", te.Name)
		return
	} else if te.Begin < 0 || te.End > len(te.Source) || te.Begin > te.End {
		fmt.Fprintf(w, "%s, invalid position %d-%d", te.Name, te.Begin, te.End)
		return
	}

	before, culprit, after := bca(te.Source, te.Begin, te.End)
	// Find the part of "before" that is on the same line as the culprit.
	lineBefore := lastLine(before)
	// Find on which line the culprit begins.
	beginLine := strings.Count(before, "\n") + 1
	// Find on which column the culprit begins.
	beginCol := countRunes(lineBefore) + 1

	// If the culprit ends with a newline, stripe it. Otherwise stick the part
	// of "after" that is on the same line of the last line of the culprit.
	var lineAfter string
	if strings.HasSuffix(culprit, "\n") {
		culprit = culprit[:len(culprit)-1]
	} else {
		lineAfter = firstLine(after)
	}

	// Find on which line and column the culprit ends.
	endLine := beginLine + strings.Count(culprit, "\n")
	var endCol int
	if endLine == beginLine {
		endCol = beginCol + countRunes(culprit) - 1
	} else {
		endCol = countRunes(lastLine(culprit))
	}

	if beginLine == endLine {
		if endCol <= beginCol {
			fmt.Fprintf(w, "%s, line %d, col %d:\n", te.Name, beginLine, beginCol)
		} else {
			fmt.Fprintf(w, "%s, line %d, col %d-%d:\n", te.Name, beginLine, beginCol, endCol)
		}
	} else {
		fmt.Fprintf(w, "%s, line %d col %d - line %d col %d:\n", te.Name, beginLine, beginCol, endLine, endCol)
	}

	fmt.Fprintf(w, "%s%s", sourceIndent, lineBefore)

	for i, line := range strings.Split(culprit, "\n") {
		if i > 0 {
			fmt.Fprintf(w, "\n%s", sourceIndent)
		}
		fmt.Fprintf(w, "\033[%sm%s\033[m", CulpritStyle, line)
	}

	fmt.Fprintf(w, "%s", lineAfter)
}

func bca(s string, a, b int) (string, string, string) {
	return s[:a], s[a:b], s[b:]
}

func countRunes(s string) int {
	return strings.Count(s, "") - 1
}

func firstLine(s string) string {
	i := strings.IndexByte(s, '\n')
	if i == -1 {
		return s
	}
	return s[:i]
}

func lastLine(s string) string {
	// When s does not contain '\n', LastIndexByte returns -1, which happens to
	// be what we want.
	return s[strings.LastIndexByte(s, '\n')+1:]
}
<commit_msg>Don't show column number in traceback.<commit_after>package util

import (
	"fmt"
	"io"
	"strings"
)

type Traceback struct {
	Name   string
	Source string
	Begin  int
	End    int
	Next   *Traceback
}

var CulpritStyle = "1;4"

func (te *Traceback) Pprint(w io.Writer, sourceIndent string) {
	if te.Begin == -1 {
		fmt.Fprintf(w, "%s, unknown position", te.Name)
		return
	} else if te.Begin < 0 || te.End > len(te.Source) || te.Begin > te.End {
		fmt.Fprintf(w, "%s, invalid position %d-%d", te.Name, te.Begin, te.End)
		return
	}

	before, culprit, after := bca(te.Source, te.Begin, te.End)
	// Find the part of "before" that is on the same line as the culprit.
	lineBefore := lastLine(before)
	// Find on which line the culprit begins.
	beginLine := strings.Count(before, "\n") + 1

	// If the culprit ends with a newline, stripe it. Otherwise stick the part
	// of "after" that is on the same line of the last line of the culprit.
	var lineAfter string
	if strings.HasSuffix(culprit, "\n") {
		culprit = culprit[:len(culprit)-1]
	} else {
		lineAfter = firstLine(after)
	}

	// Find on which line and column the culprit ends.
	endLine := beginLine + strings.Count(culprit, "\n")

	if beginLine == endLine {
		fmt.Fprintf(w, "%s, line %d:\n", te.Name, beginLine)
	} else {
		fmt.Fprintf(w, "%s, line %d-%d:\n", te.Name, beginLine, endLine)
	}

	fmt.Fprintf(w, "%s%s", sourceIndent, lineBefore)

	if culprit == "" {
		culprit = "^"
	}
	for i, line := range strings.Split(culprit, "\n") {
		if i > 0 {
			fmt.Fprintf(w, "\n%s", sourceIndent)
		}
		fmt.Fprintf(w, "\033[%sm%s\033[m", CulpritStyle, line)
	}

	fmt.Fprintf(w, "%s", lineAfter)
}

func bca(s string, a, b int) (string, string, string) {
	return s[:a], s[a:b], s[b:]
}

func countRunes(s string) int {
	return strings.Count(s, "") - 1
}

func firstLine(s string) string {
	i := strings.IndexByte(s, '\n')
	if i == -1 {
		return s
	}
	return s[:i]
}

func lastLine(s string) string {
	// When s does not contain '\n', LastIndexByte returns -1, which happens to
	// be what we want.
	return s[strings.LastIndexByte(s, '\n')+1:]
}
<|endoftext|>


<commit_before>package agent

import (
	"bufio"
	"errors"
	"net"
	"net/http"
	"net/url"
	"runtime"
	"time"

	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/logger"
)

var debug = false

type APIClient struct {
	Endpoint string
	Token    string
}

func APIClientEnableHTTPDebug() {
	debug = true
}

func (a APIClient) Create() *api.Client {
	u, err := url.Parse(a.Endpoint)
	if err != nil {
		logger.Warn("Failed to parse %q: %v", a.Endpoint, err)
	}

	if u != nil && u.Scheme == `unix` {
		return a.createFromSocket(u.Path)
	}

	httpTransport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).DialContext,
		MaxIdleConns:        100,
		IdleConnTimeout:     90 * time.Second,
		TLSHandshakeTimeout: 30 * time.Second,
	}

	// Configure the HTTP client
	httpClient := &http.Client{Transport: &api.AuthenticatedTransport{
		Token:     a.Token,
		Transport: httpTransport,
	}}
	httpClient.Timeout = 60 * time.Second

	// Create the Buildkite Agent API Client
	client := api.NewClient(httpClient)
	client.BaseURL, _ = url.Parse(a.Endpoint)
	client.UserAgent = a.UserAgent()
	client.DebugHTTP = debug

	return client
}

func (a APIClient) createFromSocket(socket string) *api.Client {
	httpClient := &http.Client{
		Transport: &api.AuthenticatedTransport{
			Token: a.Token,
			Transport: &socketTransport{
				Socket:      socket,
				DialTimeout: 30 * time.Second,
			},
		},
	}

	// Create the Buildkite Agent API Client
	client := api.NewClient(httpClient)
	client.BaseURL, _ = url.Parse(`http+unix://buildkite-agent`)
	client.UserAgent = a.UserAgent()
	client.DebugHTTP = debug

	return client
}

func (a APIClient) UserAgent() string {
	return "buildkite-agent/" + Version() + "." + BuildVersion() + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"
}

// Transport is a http.RoundTripper that connects to Unix domain sockets.
type socketTransport struct {
	DialTimeout           time.Duration
	RequestTimeout        time.Duration
	ResponseHeaderTimeout time.Duration
	Socket                string
}

// RoundTrip executes a single HTTP transaction. See net/http.RoundTripper.
func (t *socketTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	if req.URL == nil {
		return nil, errors.New("http+unix: nil Request.URL")
	}
	if req.URL.Scheme != `http+unix` {
		return nil, errors.New("unsupported protocol scheme: " + req.URL.Scheme)
	}
	if req.URL.Host == "" {
		return nil, errors.New("http+unix: no Host in request URL")
	}

	c, err := net.DialTimeout("unix", t.Socket, t.DialTimeout)
	if err != nil {
		return nil, err
	}
	r := bufio.NewReader(c)
	if t.RequestTimeout > 0 {
		c.SetWriteDeadline(time.Now().Add(t.RequestTimeout))
	}
	if err := req.Write(c); err != nil {
		return nil, err
	}
	if t.ResponseHeaderTimeout > 0 {
		c.SetReadDeadline(time.Now().Add(t.ResponseHeaderTimeout))
	}
	resp, err := http.ReadResponse(r, req)
	return resp, err
}
<commit_msg>Add back in explicit defaults for transport settings<commit_after>package agent

import (
	"bufio"
	"errors"
	"net"
	"net/http"
	"net/url"
	"runtime"
	"time"

	"github.com/buildkite/agent/api"
	"github.com/buildkite/agent/logger"
)

var debug = false

type APIClient struct {
	Endpoint string
	Token    string
}

func APIClientEnableHTTPDebug() {
	debug = true
}

func (a APIClient) Create() *api.Client {
	u, err := url.Parse(a.Endpoint)
	if err != nil {
		logger.Warn("Failed to parse %q: %v", a.Endpoint, err)
	}

	if u != nil && u.Scheme == `unix` {
		return a.createFromSocket(u.Path)
	}

	httpTransport := &http.Transport{
		Proxy:              http.ProxyFromEnvironment,
		DisableCompression: false,
		DisableKeepAlives:  false,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).DialContext,
		MaxIdleConns:        100,
		IdleConnTimeout:     90 * time.Second,
		TLSHandshakeTimeout: 30 * time.Second,
	}

	// Configure the HTTP client
	httpClient := &http.Client{Transport: &api.AuthenticatedTransport{
		Token:     a.Token,
		Transport: httpTransport,
	}}
	httpClient.Timeout = 60 * time.Second

	// Create the Buildkite Agent API Client
	client := api.NewClient(httpClient)
	client.BaseURL, _ = url.Parse(a.Endpoint)
	client.UserAgent = a.UserAgent()
	client.DebugHTTP = debug

	return client
}

func (a APIClient) createFromSocket(socket string) *api.Client {
	httpClient := &http.Client{
		Transport: &api.AuthenticatedTransport{
			Token: a.Token,
			Transport: &socketTransport{
				Socket:      socket,
				DialTimeout: 30 * time.Second,
			},
		},
	}

	// Create the Buildkite Agent API Client
	client := api.NewClient(httpClient)
	client.BaseURL, _ = url.Parse(`http+unix://buildkite-agent`)
	client.UserAgent = a.UserAgent()
	client.DebugHTTP = debug

	return client
}

func (a APIClient) UserAgent() string {
	return "buildkite-agent/" + Version() + "." + BuildVersion() + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"
}

// Transport is a http.RoundTripper that connects to Unix domain sockets.
type socketTransport struct {
	DialTimeout           time.Duration
	RequestTimeout        time.Duration
	ResponseHeaderTimeout time.Duration
	Socket                string
}

// RoundTrip executes a single HTTP transaction. See net/http.RoundTripper.
func (t *socketTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	if req.URL == nil {
		return nil, errors.New("http+unix: nil Request.URL")
	}
	if req.URL.Scheme != `http+unix` {
		return nil, errors.New("unsupported protocol scheme: " + req.URL.Scheme)
	}
	if req.URL.Host == "" {
		return nil, errors.New("http+unix: no Host in request URL")
	}

	c, err := net.DialTimeout("unix", t.Socket, t.DialTimeout)
	if err != nil {
		return nil, err
	}
	r := bufio.NewReader(c)
	if t.RequestTimeout > 0 {
		c.SetWriteDeadline(time.Now().Add(t.RequestTimeout))
	}
	if err := req.Write(c); err != nil {
		return nil, err
	}
	if t.ResponseHeaderTimeout > 0 {
		c.SetReadDeadline(time.Now().Add(t.ResponseHeaderTimeout))
	}
	resp, err := http.ReadResponse(r, req)
	return resp, err
}
<|endoftext|>


<commit_before>package speedtest

import (
	"fmt"
	"io"
	"net"
	"time"

	"github.com/dustin/go-humanize"
)

type Stats struct {
	Bytes, Blocks int64
	Elapsed       time.Duration
	Overhead      time.Duration
	// data source didn't have a new block ready, previous buffer was reused.
	Repeats int64
	// data sink couldn't receive read data, block dropped
	Dropped int64
}

func Provide(c net.Conn, dataSource <-chan []byte, doneChunks chan<- []byte, stats chan<- Stats) error {
	data := <-dataSource

	var s Stats
	start := time.Now()
	overhead := start
	for {
		s.Overhead += time.Since(overhead)
		n, err := c.Write(data)
		overhead = time.Now()

		if err != nil {
			return err
		}
		s.Bytes += int64(n)
		s.Blocks++
		s.Elapsed = time.Since(start)

		select {
		case stats <- s:
		default:
		}

		select {
		case doneChunks <- data:
		default:
			s.Dropped++
		}

		select {
		case newData, open := <-dataSource:
			if !open {
				return io.EOF
			}
			data = newData
		default:
			s.Repeats++
		}
	}
}

func Consume(c net.Conn, blockSource <-chan []byte, blockSink chan<- []byte, stats chan<- Stats) error {
	buf := <-blockSource

	var s Stats
	start := time.Now()
	overhead := start
	for {
		s.Overhead += time.Since(overhead)
		n, err := c.Read(buf)
		overhead = time.Now()

		if err != nil {
			return err
		}
		s.Bytes += int64(n)
		s.Blocks++
		s.Elapsed = time.Since(start)

		select {
		case stats <- s:
		default:
		}

		select {
		case blockSink <- buf:
		default:
			s.Dropped++
		}

		select {
		case newBuf, open := <-blockSource:
			if !open {
				return io.EOF
			}
			buf = newBuf
		default:
			s.Repeats++
		}
	}
}

func Report(prefix string, rate time.Duration, stats <-chan Stats) {
	for _ = range time.NewTicker(rate).C {
		s, open := <-stats
		if !open {
			return
		}
		if s.Elapsed == 0 {
			continue
		}
		Bps := uint64(float64(s.Bytes) / (s.Elapsed.Seconds()))
		fmt.Printf("%s: %s in %s: %sps  [%s, %d repeats, %d drops, %v overhead]\n",
			prefix,
			humanize.Bytes(uint64(s.Bytes)), s.Elapsed, humanize.Bytes(Bps),
			humanize.SI(float64(s.Blocks), " blocks"),
			s.Repeats, s.Dropped, s.Overhead)
	}
}
<commit_msg>Improve reporting to show instaneous speed in bps (not Bps).<commit_after>package speedtest

import (
	"fmt"
	"io"
	"net"
	"strings"
	"time"

	"github.com/dustin/go-humanize"
)

type Stats struct {
	Bytes, Blocks int64
	Elapsed       time.Duration
	Overhead      time.Duration
	// data source didn't have a new block ready, previous buffer was reused.
	Repeats int64
	// data sink couldn't receive read data, block dropped
	Dropped int64
}

func (s Stats) Since(s2 Stats) Stats {
	return Stats{
		Bytes:    s.Bytes - s2.Bytes,
		Blocks:   s.Blocks - s2.Blocks,
		Elapsed:  s.Elapsed - s2.Elapsed,
		Overhead: s.Overhead - s2.Overhead,
		Repeats:  s.Repeats - s2.Repeats,
		Dropped:  s.Dropped - s2.Dropped,
	}
}

func Provide(c net.Conn, dataSource <-chan []byte, doneChunks chan<- []byte, stats chan<- Stats) error {
	data := <-dataSource

	var s Stats
	start := time.Now()
	overhead := start
	for {
		s.Overhead += time.Since(overhead)
		n, err := c.Write(data)
		overhead = time.Now()

		if err != nil {
			return err
		}
		s.Bytes += int64(n)
		s.Blocks++
		s.Elapsed = time.Since(start)

		select {
		case stats <- s:
		default:
		}

		select {
		case doneChunks <- data:
		default:
			s.Dropped++
		}

		select {
		case newData, open := <-dataSource:
			if !open {
				return io.EOF
			}
			data = newData
		default:
			s.Repeats++
		}
	}
}

func Consume(c net.Conn, blockSource <-chan []byte, blockSink chan<- []byte, stats chan<- Stats) error {
	buf := <-blockSource

	var s Stats
	start := time.Now()
	overhead := start
	for {
		s.Overhead += time.Since(overhead)
		n, err := c.Read(buf)
		overhead = time.Now()

		if err != nil {
			return err
		}
		s.Bytes += int64(n)
		s.Blocks++
		s.Elapsed = time.Since(start)

		select {
		case stats <- s:
		default:
		}

		select {
		case blockSink <- buf:
		default:
			s.Dropped++
		}

		select {
		case newBuf, open := <-blockSource:
			if !open {
				return io.EOF
			}
			buf = newBuf
		default:
			s.Repeats++
		}
	}
}

func Report(prefix string, rate time.Duration, stats <-chan Stats) {
	var last Stats
	for _ = range time.NewTicker(rate).C {
		s, open := <-stats
		if !open {
			return
		}
		if s.Elapsed == 0 {
			continue
		}
		last, s = s, s.Since(last)
		Bps := uint64(float64(s.Bytes) / (s.Elapsed.Seconds()))
		fmt.Printf("%s: %s in %s: %sbps  [%s, %d repeats, %d drops, %v overhead]\n",
			prefix,
			humanize.Bytes(uint64(s.Bytes)),
			s.Elapsed,
			strings.TrimSuffix(humanize.Bytes(8*Bps), "B"),
			humanize.SI(float64(s.Blocks), " blocks"),
			s.Repeats, s.Dropped, s.Overhead)
	}
}
<|endoftext|>


<commit_before>package shared

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"gopkg.in/robfig/cron.v2"

	"github.com/lxc/lxd/shared/units"
)

type ContainerAction string

const (
	Stop     ContainerAction = "stop"
	Start    ContainerAction = "start"
	Restart  ContainerAction = "restart"
	Freeze   ContainerAction = "freeze"
	Unfreeze ContainerAction = "unfreeze"
)

func IsInt64(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s", value)
	}

	return nil
}

func IsUint8(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseUint(value, 10, 8)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s. Must be between 0 and 255", value)
	}

	return nil
}

func IsUint32(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseUint(value, 10, 32)
	if err != nil {
		return fmt.Errorf("Invalid value for uint32: %s: %v", value, err)
	}

	return nil
}

func IsPriority(value string) error {
	if value == "" {
		return nil
	}

	valueInt, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s", value)
	}

	if valueInt < 0 || valueInt > 10 {
		return fmt.Errorf("Invalid value for a limit '%s'. Must be between 0 and 10", value)
	}

	return nil
}

func IsBool(value string) error {
	if value == "" {
		return nil
	}

	if !StringInSlice(strings.ToLower(value), []string{"true", "false", "yes", "no", "1", "0", "on", "off"}) {
		return fmt.Errorf("Invalid value for a boolean: %s", value)
	}

	return nil
}

func IsOneOf(value string, valid []string) error {
	if value == "" {
		return nil
	}

	if !StringInSlice(value, valid) {
		return fmt.Errorf("Invalid value: %s (not one of %s)", value, valid)
	}

	return nil
}

func IsAny(value string) error {
	return nil
}

func IsNotEmpty(value string) error {
	if value == "" {
		return fmt.Errorf("Required value")
	}

	return nil
}

func IsUnixUserID(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseUint(value, 10, 32)
	if err != nil {
		return fmt.Errorf("Invalid value for a UNIX ID")
	}

	return nil
}

func IsOctalFileMode(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseUint(value, 8, 32)
	if err != nil {
		return fmt.Errorf("Invalid value for an octal file mode")
	}

	return nil
}

// IsDeviceID validates string is four lowercase hex characters suitable as Vendor or Device ID.
func IsDeviceID(value string) error {
	if value == "" {
		return nil
	}

	regexHexLc, err := regexp.Compile("^[0-9a-f]+$")
	if err != nil {
		return err
	}

	if len(value) != 4 || !regexHexLc.MatchString(value) {
		return fmt.Errorf("Invalid value, must be four lower case hex characters")
	}

	return nil
}

// IsRootDiskDevice returns true if the given device representation is
// configured as root disk for a container. It typically get passed a specific
// entry of api.Container.Devices.
func IsRootDiskDevice(device map[string]string) bool {
	if device["type"] == "disk" && device["path"] == "/" && device["source"] == "" {
		return true
	}

	return false
}

// GetRootDiskDevice returns the container device that is configured as root disk
func GetRootDiskDevice(devices map[string]map[string]string) (string, map[string]string, error) {
	var devName string
	var dev map[string]string

	for n, d := range devices {
		if IsRootDiskDevice(d) {
			if devName != "" {
				return "", nil, fmt.Errorf("More than one root device found")
			}

			devName = n
			dev = d
		}
	}

	if devName != "" {
		return devName, dev, nil
	}

	return "", nil, fmt.Errorf("No root device could be found")
}

// KnownContainerConfigKeys maps all fully defined, well-known config keys
// to an appropriate checker function, which validates whether or not a
// given value is syntactically legal.
var KnownContainerConfigKeys = map[string]func(value string) error{
	"boot.autostart":             IsBool,
	"boot.autostart.delay":       IsInt64,
	"boot.autostart.priority":    IsInt64,
	"boot.stop.priority":         IsInt64,
	"boot.host_shutdown_timeout": IsInt64,

	"limits.cpu": func(value string) error {
		if value == "" {
			return nil
		}

		// Validate the character set
		match, _ := regexp.MatchString("^[-,0-9]*$", value)
		if !match {
			return fmt.Errorf("Invalid CPU limit syntax")
		}

		// Validate first character
		if strings.HasPrefix(value, "-") || strings.HasPrefix(value, ",") {
			return fmt.Errorf("CPU limit can't start with a separator")
		}

		// Validate last character
		if strings.HasSuffix(value, "-") || strings.HasSuffix(value, ",") {
			return fmt.Errorf("CPU limit can't end with a separator")
		}

		return nil
	},
	"limits.cpu.allowance": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			// Percentage based allocation
			_, err := strconv.Atoi(strings.TrimSuffix(value, "%"))
			if err != nil {
				return err
			}

			return nil
		}

		// Time based allocation
		fields := strings.SplitN(value, "/", 2)
		if len(fields) != 2 {
			return fmt.Errorf("Invalid allowance: %s", value)
		}

		_, err := strconv.Atoi(strings.TrimSuffix(fields[0], "ms"))
		if err != nil {
			return err
		}

		_, err = strconv.Atoi(strings.TrimSuffix(fields[1], "ms"))
		if err != nil {
			return err
		}

		return nil
	},
	"limits.cpu.priority": IsPriority,

	"limits.disk.priority": IsPriority,

	"limits.memory": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			_, err := strconv.ParseInt(strings.TrimSuffix(value, "%"), 10, 64)
			if err != nil {
				return err
			}

			return nil
		}

		_, err := units.ParseByteSizeString(value)
		if err != nil {
			return err
		}

		return nil
	},
	"limits.memory.enforce": func(value string) error {
		return IsOneOf(value, []string{"soft", "hard"})
	},
	"limits.memory.swap":          IsBool,
	"limits.memory.swap.priority": IsPriority,

	"limits.network.priority": IsPriority,

	"limits.processes": IsInt64,

	"linux.kernel_modules": IsAny,

	"migration.incremental.memory":            IsBool,
	"migration.incremental.memory.iterations": IsUint32,
	"migration.incremental.memory.goal":       IsUint32,

	"nvidia.runtime":             IsBool,
	"nvidia.driver.capabilities": IsAny,
	"nvidia.require.cuda":        IsAny,
	"nvidia.require.driver":      IsAny,

	"security.nesting":       IsBool,
	"security.privileged":    IsBool,
	"security.devlxd":        IsBool,
	"security.devlxd.images": IsBool,

	"security.protection.delete": IsBool,
	"security.protection.shift":  IsBool,

	"security.idmap.base":     IsUint32,
	"security.idmap.isolated": IsBool,
	"security.idmap.size":     IsUint32,

	"security.syscalls.blacklist_default":  IsBool,
	"security.syscalls.blacklist_compat":   IsBool,
	"security.syscalls.blacklist":          IsAny,
	"security.syscalls.intercept.mknod":    IsBool,
	"security.syscalls.intercept.setxattr": IsBool,
	"security.syscalls.whitelist":          IsAny,

	"snapshots.schedule": func(value string) error {
		if value == "" {
			return nil
		}

		if len(strings.Split(value, " ")) != 5 {
			return fmt.Errorf("Schedule must be of the form: <minute> <hour> <day-of-month> <month> <day-of-week>")
		}

		_, err := cron.Parse(fmt.Sprintf("* %s", value))
		if err != nil {
			return errors.Wrap(err, "Error parsing schedule")
		}

		return nil
	},
	"snapshots.schedule.stopped": IsBool,
	"snapshots.pattern":          IsAny,
	"snapshots.expiry": func(value string) error {
		// Validate expression
		_, err := GetSnapshotExpiry(time.Time{}, value)
		return err
	},

	// Caller is responsible for full validation of any raw.* value
	"raw.apparmor": IsAny,
	"raw.lxc":      IsAny,
	"raw.seccomp":  IsAny,
	"raw.idmap":    IsAny,

	"volatile.apply_template":   IsAny,
	"volatile.base_image":       IsAny,
	"volatile.last_state.idmap": IsAny,
	"volatile.last_state.power": IsAny,
	"volatile.idmap.base":       IsAny,
	"volatile.idmap.current":    IsAny,
	"volatile.idmap.next":       IsAny,
	"volatile.apply_quota":      IsAny,
}

// ConfigKeyChecker returns a function that will check whether or not
// a provide value is valid for the associate config key.  Returns an
// error if the key is not known.  The checker function only performs
// syntactic checking of the value, semantic and usage checking must
// be done by the caller.  User defined keys are always considered to
// be valid, e.g. user.* and environment.* keys.
func ConfigKeyChecker(key string) (func(value string) error, error) {
	if f, ok := KnownContainerConfigKeys[key]; ok {
		return f, nil
	}

	if strings.HasPrefix(key, "volatile.") {
		if strings.HasSuffix(key, ".hwaddr") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".name") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".host_name") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".mtu") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".created") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".id") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".vlan") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".spoofcheck") {
			return IsAny, nil
		}
	}

	if strings.HasPrefix(key, "environment.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "user.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "image.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "limits.kernel.") &&
		(len(key) > len("limits.kernel.")) {
		return IsAny, nil
	}

	return nil, fmt.Errorf("Unknown configuration key: %s", key)
}

// ContainerGetParentAndSnapshotName returns the parent container name, snapshot
// name, and whether it actually was a snapshot name.
func ContainerGetParentAndSnapshotName(name string) (string, string, bool) {
	fields := strings.SplitN(name, SnapshotDelimiter, 2)
	if len(fields) == 1 {
		return name, "", false
	}

	return fields[0], fields[1], true
}
<commit_msg>shared/container: Removes device related validation functions<commit_after>package shared

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"gopkg.in/robfig/cron.v2"

	"github.com/lxc/lxd/shared/units"
)

type ContainerAction string

const (
	Stop     ContainerAction = "stop"
	Start    ContainerAction = "start"
	Restart  ContainerAction = "restart"
	Freeze   ContainerAction = "freeze"
	Unfreeze ContainerAction = "unfreeze"
)

func IsInt64(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s", value)
	}

	return nil
}

func IsUint8(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseUint(value, 10, 8)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s. Must be between 0 and 255", value)
	}

	return nil
}

func IsUint32(value string) error {
	if value == "" {
		return nil
	}

	_, err := strconv.ParseUint(value, 10, 32)
	if err != nil {
		return fmt.Errorf("Invalid value for uint32: %s: %v", value, err)
	}

	return nil
}

func IsPriority(value string) error {
	if value == "" {
		return nil
	}

	valueInt, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		return fmt.Errorf("Invalid value for an integer: %s", value)
	}

	if valueInt < 0 || valueInt > 10 {
		return fmt.Errorf("Invalid value for a limit '%s'. Must be between 0 and 10", value)
	}

	return nil
}

func IsBool(value string) error {
	if value == "" {
		return nil
	}

	if !StringInSlice(strings.ToLower(value), []string{"true", "false", "yes", "no", "1", "0", "on", "off"}) {
		return fmt.Errorf("Invalid value for a boolean: %s", value)
	}

	return nil
}

func IsOneOf(value string, valid []string) error {
	if value == "" {
		return nil
	}

	if !StringInSlice(value, valid) {
		return fmt.Errorf("Invalid value: %s (not one of %s)", value, valid)
	}

	return nil
}

func IsAny(value string) error {
	return nil
}

func IsNotEmpty(value string) error {
	if value == "" {
		return fmt.Errorf("Required value")
	}

	return nil
}

// IsDeviceID validates string is four lowercase hex characters suitable as Vendor or Device ID.
func IsDeviceID(value string) error {
	if value == "" {
		return nil
	}

	regexHexLc, err := regexp.Compile("^[0-9a-f]+$")
	if err != nil {
		return err
	}

	if len(value) != 4 || !regexHexLc.MatchString(value) {
		return fmt.Errorf("Invalid value, must be four lower case hex characters")
	}

	return nil
}

// IsRootDiskDevice returns true if the given device representation is
// configured as root disk for a container. It typically get passed a specific
// entry of api.Container.Devices.
func IsRootDiskDevice(device map[string]string) bool {
	if device["type"] == "disk" && device["path"] == "/" && device["source"] == "" {
		return true
	}

	return false
}

// GetRootDiskDevice returns the container device that is configured as root disk
func GetRootDiskDevice(devices map[string]map[string]string) (string, map[string]string, error) {
	var devName string
	var dev map[string]string

	for n, d := range devices {
		if IsRootDiskDevice(d) {
			if devName != "" {
				return "", nil, fmt.Errorf("More than one root device found")
			}

			devName = n
			dev = d
		}
	}

	if devName != "" {
		return devName, dev, nil
	}

	return "", nil, fmt.Errorf("No root device could be found")
}

// KnownContainerConfigKeys maps all fully defined, well-known config keys
// to an appropriate checker function, which validates whether or not a
// given value is syntactically legal.
var KnownContainerConfigKeys = map[string]func(value string) error{
	"boot.autostart":             IsBool,
	"boot.autostart.delay":       IsInt64,
	"boot.autostart.priority":    IsInt64,
	"boot.stop.priority":         IsInt64,
	"boot.host_shutdown_timeout": IsInt64,

	"limits.cpu": func(value string) error {
		if value == "" {
			return nil
		}

		// Validate the character set
		match, _ := regexp.MatchString("^[-,0-9]*$", value)
		if !match {
			return fmt.Errorf("Invalid CPU limit syntax")
		}

		// Validate first character
		if strings.HasPrefix(value, "-") || strings.HasPrefix(value, ",") {
			return fmt.Errorf("CPU limit can't start with a separator")
		}

		// Validate last character
		if strings.HasSuffix(value, "-") || strings.HasSuffix(value, ",") {
			return fmt.Errorf("CPU limit can't end with a separator")
		}

		return nil
	},
	"limits.cpu.allowance": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			// Percentage based allocation
			_, err := strconv.Atoi(strings.TrimSuffix(value, "%"))
			if err != nil {
				return err
			}

			return nil
		}

		// Time based allocation
		fields := strings.SplitN(value, "/", 2)
		if len(fields) != 2 {
			return fmt.Errorf("Invalid allowance: %s", value)
		}

		_, err := strconv.Atoi(strings.TrimSuffix(fields[0], "ms"))
		if err != nil {
			return err
		}

		_, err = strconv.Atoi(strings.TrimSuffix(fields[1], "ms"))
		if err != nil {
			return err
		}

		return nil
	},
	"limits.cpu.priority": IsPriority,

	"limits.disk.priority": IsPriority,

	"limits.memory": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			_, err := strconv.ParseInt(strings.TrimSuffix(value, "%"), 10, 64)
			if err != nil {
				return err
			}

			return nil
		}

		_, err := units.ParseByteSizeString(value)
		if err != nil {
			return err
		}

		return nil
	},
	"limits.memory.enforce": func(value string) error {
		return IsOneOf(value, []string{"soft", "hard"})
	},
	"limits.memory.swap":          IsBool,
	"limits.memory.swap.priority": IsPriority,

	"limits.network.priority": IsPriority,

	"limits.processes": IsInt64,

	"linux.kernel_modules": IsAny,

	"migration.incremental.memory":            IsBool,
	"migration.incremental.memory.iterations": IsUint32,
	"migration.incremental.memory.goal":       IsUint32,

	"nvidia.runtime":             IsBool,
	"nvidia.driver.capabilities": IsAny,
	"nvidia.require.cuda":        IsAny,
	"nvidia.require.driver":      IsAny,

	"security.nesting":       IsBool,
	"security.privileged":    IsBool,
	"security.devlxd":        IsBool,
	"security.devlxd.images": IsBool,

	"security.protection.delete": IsBool,
	"security.protection.shift":  IsBool,

	"security.idmap.base":     IsUint32,
	"security.idmap.isolated": IsBool,
	"security.idmap.size":     IsUint32,

	"security.syscalls.blacklist_default":  IsBool,
	"security.syscalls.blacklist_compat":   IsBool,
	"security.syscalls.blacklist":          IsAny,
	"security.syscalls.intercept.mknod":    IsBool,
	"security.syscalls.intercept.setxattr": IsBool,
	"security.syscalls.whitelist":          IsAny,

	"snapshots.schedule": func(value string) error {
		if value == "" {
			return nil
		}

		if len(strings.Split(value, " ")) != 5 {
			return fmt.Errorf("Schedule must be of the form: <minute> <hour> <day-of-month> <month> <day-of-week>")
		}

		_, err := cron.Parse(fmt.Sprintf("* %s", value))
		if err != nil {
			return errors.Wrap(err, "Error parsing schedule")
		}

		return nil
	},
	"snapshots.schedule.stopped": IsBool,
	"snapshots.pattern":          IsAny,
	"snapshots.expiry": func(value string) error {
		// Validate expression
		_, err := GetSnapshotExpiry(time.Time{}, value)
		return err
	},

	// Caller is responsible for full validation of any raw.* value
	"raw.apparmor": IsAny,
	"raw.lxc":      IsAny,
	"raw.seccomp":  IsAny,
	"raw.idmap":    IsAny,

	"volatile.apply_template":   IsAny,
	"volatile.base_image":       IsAny,
	"volatile.last_state.idmap": IsAny,
	"volatile.last_state.power": IsAny,
	"volatile.idmap.base":       IsAny,
	"volatile.idmap.current":    IsAny,
	"volatile.idmap.next":       IsAny,
	"volatile.apply_quota":      IsAny,
}

// ConfigKeyChecker returns a function that will check whether or not
// a provide value is valid for the associate config key.  Returns an
// error if the key is not known.  The checker function only performs
// syntactic checking of the value, semantic and usage checking must
// be done by the caller.  User defined keys are always considered to
// be valid, e.g. user.* and environment.* keys.
func ConfigKeyChecker(key string) (func(value string) error, error) {
	if f, ok := KnownContainerConfigKeys[key]; ok {
		return f, nil
	}

	if strings.HasPrefix(key, "volatile.") {
		if strings.HasSuffix(key, ".hwaddr") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".name") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".host_name") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".mtu") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".created") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".id") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".vlan") {
			return IsAny, nil
		}

		if strings.HasSuffix(key, ".spoofcheck") {
			return IsAny, nil
		}
	}

	if strings.HasPrefix(key, "environment.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "user.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "image.") {
		return IsAny, nil
	}

	if strings.HasPrefix(key, "limits.kernel.") &&
		(len(key) > len("limits.kernel.")) {
		return IsAny, nil
	}

	return nil, fmt.Errorf("Unknown configuration key: %s", key)
}

// ContainerGetParentAndSnapshotName returns the parent container name, snapshot
// name, and whether it actually was a snapshot name.
func ContainerGetParentAndSnapshotName(name string) (string, string, bool) {
	fields := strings.SplitN(name, SnapshotDelimiter, 2)
	if len(fields) == 1 {
		return name, "", false
	}

	return fields[0], fields[1], true
}
<|endoftext|>