<commit_before>package storage

import (
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"sync"
)

type Version uint8

const (
	SuperBlockSize = 8
	CurrentVersion = Version(1)
)

type Volume struct {
	Id       VolumeId
	dir      string
	dataFile *os.File
	nm       *NeedleMap

	replicaType ReplicationType
	version     Version

	accessLock sync.Mutex
}

func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume) {
	v = &Volume{dir: dirname, Id: id, replicaType: replicationType}
	v.load()
	return
}
func (v *Volume) load() error {
	var e error
	fileName := path.Join(v.dir, v.Id.String())
	v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
	if e != nil {
		return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
	}
	if v.replicaType == CopyNil {
		if e = v.readSuperBlock(); e != nil {
			return e
		}
	} else {
		v.maybeWriteSuperBlock()
	}
	indexFile, ie := os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644)
	if ie != nil {
		return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
	}
	v.nm = LoadNeedleMap(indexFile)
	return nil
}
func (v *Volume) Size() int64 {
    v.accessLock.Lock()
    defer v.accessLock.Unlock()
	stat, e := v.dataFile.Stat()
	if e == nil {
		return stat.Size()
	}
	fmt.Printf("Failed to read file size %s %s\n", v.dataFile.Name(), e.Error())
	return -1
}
func (v *Volume) Close() {
    v.accessLock.Lock()
    defer v.accessLock.Unlock()
	v.nm.Close()
	v.dataFile.Close()
}
func (v *Volume) maybeWriteSuperBlock() {
	stat, _ := v.dataFile.Stat()
	if stat.Size() == 0 {
		header := make([]byte, SuperBlockSize)
		header[0] = byte(CurrentVersion)
		header[1] = v.replicaType.Byte()
		v.dataFile.Write(header)
	}
}
func (v *Volume) readSuperBlock() error {
	v.dataFile.Seek(0, 0)
	header := make([]byte, SuperBlockSize)
	if _, e := v.dataFile.Read(header); e != nil {
		return fmt.Errorf("cannot read superblock: %s", e)
	}
	v.version = Version(header[0])
	var err error
	if v.replicaType, err = NewReplicationTypeFromByte(header[1]); err != nil {
		return fmt.Errorf("cannot read replica type: %s", err)
	}
	return nil
}
func (v *Volume) NeedToReplicate() bool {
	return v.replicaType.GetCopyCount() > 1
}

func (v *Volume) write(n *Needle) uint32 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	offset, _ := v.dataFile.Seek(0, 2)
	ret := n.Append(v.dataFile)
	nv, ok := v.nm.Get(n.Id)
	if !ok || int64(nv.Offset)*8 < offset {
		v.nm.Put(n.Id, uint32(offset/8), n.Size)
	}
	return ret
}
func (v *Volume) delete(n *Needle) uint32 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	//log.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
	if ok {
		v.nm.Delete(n.Id)
		v.dataFile.Seek(int64(nv.Offset*8), 0)
		n.Append(v.dataFile)
		return nv.Size
	}
	return 0
}

func (v *Volume) read(n *Needle) (int, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	if ok && nv.Offset > 0 {
		v.dataFile.Seek(int64(nv.Offset)*8, 0)
		return n.Read(v.dataFile, nv.Size, v.version)
	}
	return -1, errors.New("Not Found")
}

func (v *Volume) garbageLevel() float64 {
    return float64(v.nm.deletionByteCounter)/float64(v.Size())
}

func (v *Volume) compact() error {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()

	filePath := path.Join(v.dir, v.Id.String())
	return v.copyDataAndGenerateIndexFile(filePath+".dat", filePath+".cpd", filePath+".cpx")
}
func (v *Volume) commitCompact() (error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	v.dataFile.Close()
	var e error
	if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpd"), path.Join(v.dir, v.Id.String()+".dat")); e != nil {
		return e
	}
	if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpx"), path.Join(v.dir, v.Id.String()+".idx")); e != nil {
		return e
	}
	if e = v.load(); e != nil {
		return e
	}
	return nil
}

func (v *Volume) copyDataAndGenerateIndexFile(srcName, dstName, idxName string) (err error) {
	src, err := os.OpenFile(srcName, os.O_RDONLY, 0644)
	if err != nil {
		return err
	}
	defer src.Close()

	dst, err := os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer dst.Close()

	idx, err := os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer idx.Close()

	src.Seek(0, 0)
	header := make([]byte, SuperBlockSize)
	if _, error := src.Read(header); error == nil {
		dst.Write(header)
	}

	n, rest := ReadNeedle(src)
	nm := NewNeedleMap(idx)
	old_offset := uint32(SuperBlockSize)
	new_offset := uint32(SuperBlockSize)
	for n != nil {
		nv, ok := v.nm.Get(n.Id)
		//log.Println("file size is", n.Size, "rest", rest)
		if !ok || nv.Offset*8 != old_offset {
			src.Seek(int64(rest), 1)
		} else {
			if nv.Size > 0 {
				nm.Put(n.Id, new_offset/8, n.Size)
				bytes := make([]byte, n.Size+4)
				src.Read(bytes)
				n.Data = bytes[:n.Size]
				n.Checksum = NewCRC(n.Data)
				n.Append(dst)
				new_offset += rest + 16
				log.Println("saving key", n.Id, "volume offset", old_offset, "=>", new_offset, "data_size", n.Size, "rest", rest)
			}
			src.Seek(int64(rest-n.Size-4), 1)
		}
		old_offset += rest + 16
		n, rest = ReadNeedle(src)
	}

	return nil
}
<commit_msg>remove log message<commit_after>package storage

import (
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"sync"
)

type Version uint8

const (
	SuperBlockSize = 8
	CurrentVersion = Version(1)
)

type Volume struct {
	Id       VolumeId
	dir      string
	dataFile *os.File
	nm       *NeedleMap

	replicaType ReplicationType
	version     Version

	accessLock sync.Mutex
}

func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume) {
	v = &Volume{dir: dirname, Id: id, replicaType: replicationType}
	v.load()
	return
}
func (v *Volume) load() error {
	var e error
	fileName := path.Join(v.dir, v.Id.String())
	v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
	if e != nil {
		return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
	}
	if v.replicaType == CopyNil {
		if e = v.readSuperBlock(); e != nil {
			return e
		}
	} else {
		v.maybeWriteSuperBlock()
	}
	indexFile, ie := os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644)
	if ie != nil {
		return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
	}
	v.nm = LoadNeedleMap(indexFile)
	return nil
}
func (v *Volume) Size() int64 {
    v.accessLock.Lock()
    defer v.accessLock.Unlock()
	stat, e := v.dataFile.Stat()
	if e == nil {
		return stat.Size()
	}
	fmt.Printf("Failed to read file size %s %s\n", v.dataFile.Name(), e.Error())
	return -1
}
func (v *Volume) Close() {
    v.accessLock.Lock()
    defer v.accessLock.Unlock()
	v.nm.Close()
	v.dataFile.Close()
}
func (v *Volume) maybeWriteSuperBlock() {
	stat, _ := v.dataFile.Stat()
	if stat.Size() == 0 {
		header := make([]byte, SuperBlockSize)
		header[0] = byte(CurrentVersion)
		header[1] = v.replicaType.Byte()
		v.dataFile.Write(header)
	}
}
func (v *Volume) readSuperBlock() error {
	v.dataFile.Seek(0, 0)
	header := make([]byte, SuperBlockSize)
	if _, e := v.dataFile.Read(header); e != nil {
		return fmt.Errorf("cannot read superblock: %s", e)
	}
	v.version = Version(header[0])
	var err error
	if v.replicaType, err = NewReplicationTypeFromByte(header[1]); err != nil {
		return fmt.Errorf("cannot read replica type: %s", err)
	}
	return nil
}
func (v *Volume) NeedToReplicate() bool {
	return v.replicaType.GetCopyCount() > 1
}

func (v *Volume) write(n *Needle) uint32 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	offset, _ := v.dataFile.Seek(0, 2)
	ret := n.Append(v.dataFile)
	nv, ok := v.nm.Get(n.Id)
	if !ok || int64(nv.Offset)*8 < offset {
		v.nm.Put(n.Id, uint32(offset/8), n.Size)
	}
	return ret
}
func (v *Volume) delete(n *Needle) uint32 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	//log.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
	if ok {
		v.nm.Delete(n.Id)
		v.dataFile.Seek(int64(nv.Offset*8), 0)
		n.Append(v.dataFile)
		return nv.Size
	}
	return 0
}

func (v *Volume) read(n *Needle) (int, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	if ok && nv.Offset > 0 {
		v.dataFile.Seek(int64(nv.Offset)*8, 0)
		return n.Read(v.dataFile, nv.Size, v.version)
	}
	return -1, errors.New("Not Found")
}

func (v *Volume) garbageLevel() float64 {
    return float64(v.nm.deletionByteCounter)/float64(v.Size())
}

func (v *Volume) compact() error {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()

	filePath := path.Join(v.dir, v.Id.String())
	return v.copyDataAndGenerateIndexFile(filePath+".dat", filePath+".cpd", filePath+".cpx")
}
func (v *Volume) commitCompact() (error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	v.dataFile.Close()
	var e error
	if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpd"), path.Join(v.dir, v.Id.String()+".dat")); e != nil {
		return e
	}
	if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpx"), path.Join(v.dir, v.Id.String()+".idx")); e != nil {
		return e
	}
	if e = v.load(); e != nil {
		return e
	}
	return nil
}

func (v *Volume) copyDataAndGenerateIndexFile(srcName, dstName, idxName string) (err error) {
	src, err := os.OpenFile(srcName, os.O_RDONLY, 0644)
	if err != nil {
		return err
	}
	defer src.Close()

	dst, err := os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer dst.Close()

	idx, err := os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer idx.Close()

	src.Seek(0, 0)
	header := make([]byte, SuperBlockSize)
	if _, error := src.Read(header); error == nil {
		dst.Write(header)
	}

	n, rest := ReadNeedle(src)
	nm := NewNeedleMap(idx)
	old_offset := uint32(SuperBlockSize)
	new_offset := uint32(SuperBlockSize)
	for n != nil {
		nv, ok := v.nm.Get(n.Id)
		//log.Println("file size is", n.Size, "rest", rest)
		if !ok || nv.Offset*8 != old_offset {
			src.Seek(int64(rest), 1)
		} else {
			if nv.Size > 0 {
				nm.Put(n.Id, new_offset/8, n.Size)
				bytes := make([]byte, n.Size+4)
				src.Read(bytes)
				n.Data = bytes[:n.Size]
				n.Checksum = NewCRC(n.Data)
				n.Append(dst)
				new_offset += rest + 16
				//log.Println("saving key", n.Id, "volume offset", old_offset, "=>", new_offset, "data_size", n.Size, "rest", rest)
			}
			src.Seek(int64(rest-n.Size-4), 1)
		}
		old_offset += rest + 16
		n, rest = ReadNeedle(src)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package certupdater

import (
	"reflect"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/utils/set"

	"github.com/juju/juju/api/watcher"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/cert"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/network"
	"github.com/juju/juju/state"
	"github.com/juju/juju/worker"
)

var logger = loggo.GetLogger("juju.worker.certupdater")

// CertificateUpdater is responsible for generating state server certificates.
//
// In practice, CertificateUpdater is used by a state server's machine agent to watch
// that server's machines addresses in state, and write a new certificate to the
// agent's config file.
type CertificateUpdater struct {
	addressWatcher  AddressWatcher
	getter          StateServingInfoGetter
	setter          StateServingInfoSetter
	configGetter    EnvironConfigGetter
	hostPortsGetter APIHostPortsGetter
	certChanged     chan params.StateServingInfo
	addresses       []network.Address
}

// AddressWatcher is an interface that is provided to NewCertificateUpdater
// which can be used to watch for machine address changes.
type AddressWatcher interface {
	WatchAddresses() state.NotifyWatcher
	Addresses() (addresses []network.Address)
}

// EnvironConfigGetter is an interface that is provided to NewCertificateUpdater
// which can be used to get environment config.
type EnvironConfigGetter interface {
	EnvironConfig() (*config.Config, error)
}

// StateServingInfoGetter is an interface that is provided to NewCertificateUpdater
// whose StateServingInfo method will be invoked to get state serving info.
type StateServingInfoGetter interface {
	StateServingInfo() (params.StateServingInfo, bool)
}

// StateServingInfoSetter defines a function that is called to set a
// StateServingInfo value with a newly generated certificate.
type StateServingInfoSetter func(info params.StateServingInfo, done <-chan struct{}) error

// APIHostPortsGetter is an interface that is provided to NewCertificateUpdater
// whose APIHostPorts method will be invoked to get state server addresses.
type APIHostPortsGetter interface {
	APIHostPorts() ([][]network.HostPort, error)
}

// NewCertificateUpdater returns a worker.Worker that watches for changes to
// machine addresses and then generates a new state server certificate with those
// addresses in the certificate's SAN value.
func NewCertificateUpdater(addressWatcher AddressWatcher, getter StateServingInfoGetter,
	configGetter EnvironConfigGetter, hostPortsGetter APIHostPortsGetter, setter StateServingInfoSetter,
	certChanged chan params.StateServingInfo,
) worker.Worker {
	return worker.NewNotifyWorker(&CertificateUpdater{
		addressWatcher:  addressWatcher,
		configGetter:    configGetter,
		hostPortsGetter: hostPortsGetter,
		getter:          getter,
		setter:          setter,
		certChanged:     certChanged,
	})
}

// SetUp is defined on the NotifyWatchHandler interface.
func (c *CertificateUpdater) SetUp() (watcher.NotifyWatcher, error) {
	// Populate certificate SAN with any addresses we know about now.
	apiHostPorts, err := c.hostPortsGetter.APIHostPorts()
	if err != nil {
		return nil, errors.Annotate(err, "retrieving initial server addesses")
	}
	var initialSANAddresses []network.Address
	for _, server := range apiHostPorts {
		for _, nhp := range server {
			if nhp.Scope != network.ScopeCloudLocal {
				continue
			}
			initialSANAddresses = append(initialSANAddresses, nhp.Address)
		}
	}
	if err := c.updateCertificate(initialSANAddresses, make(chan struct{})); err != nil {
		return nil, errors.Annotate(err, "setting initial cerificate SAN list")
	}
	// Return
	return c.addressWatcher.WatchAddresses(), nil
}

// Handle is defined on the NotifyWatchHandler interface.
func (c *CertificateUpdater) Handle(done <-chan struct{}) error {
	addresses := c.addressWatcher.Addresses()
	if reflect.DeepEqual(addresses, c.addresses) {
		// Sometimes the watcher will tell us things have changed, when they
		// haven't as far as we can tell.
		logger.Debugf("addresses haven't really changed since last updated cert")
		return nil
	}
	return c.updateCertificate(addresses, done)
}

func (c *CertificateUpdater) updateCertificate(addresses []network.Address, done <-chan struct{}) error {
	logger.Debugf("new machine addresses: %#v", addresses)
	c.addresses = addresses

	// Older Juju deployments will not have the CA cert private key
	// available.
	stateInfo, ok := c.getter.StateServingInfo()
	if !ok {
		logger.Warningf("no state serving info, cannot regenerate server certificate")
		return nil
	}
	caPrivateKey := stateInfo.CAPrivateKey
	if caPrivateKey == "" {
		logger.Warningf("no CA cert private key, cannot regenerate server certificate")
		return nil
	}
	// Grab the env config and update a copy with ca cert private key.
	envConfig, err := c.configGetter.EnvironConfig()
	if err != nil {
		return errors.Annotate(err, "cannot read environment config")
	}
	envConfig, err = envConfig.Apply(map[string]interface{}{"ca-private-key": caPrivateKey})
	if err != nil {
		return errors.Annotate(err, "cannot add CA private key to environment config")
	}

	// For backwards compatibility, we must include "anything", "juju-apiserver"
	// and "juju-mongodb" as hostnames as that is what clients specify
	// as the hostname for verification (this certicate is used both
	// for serving MongoDB and API server connections).  We also
	// explicitly include localhost.
	serverAddrs := []string{"localhost", "juju-apiserver", "juju-mongodb", "anything"}
	for _, addr := range addresses {
		if addr.Value == "localhost" {
			continue
		}
		serverAddrs = append(serverAddrs, addr.Value)
	}
	newServerAddrs, update, err := updateRequired(stateInfo.Cert, serverAddrs)
	if err != nil {
		return errors.Annotate(err, "cannot determine if cert update needed")
	}
	if !update {
		logger.Debugf("no certificate update required")
		return nil
	}

	// Generate a new state server certificate with the machine addresses in the SAN value.
	newCert, newKey, err := envConfig.GenerateStateServerCertAndKey(newServerAddrs)
	if err != nil {
		return errors.Annotate(err, "cannot generate state server certificate")
	}
	stateInfo.Cert = string(newCert)
	stateInfo.PrivateKey = string(newKey)
	err = c.setter(stateInfo, done)
	if err != nil {
		return errors.Annotate(err, "cannot write agent config")
	}
	logger.Infof("State Server cerificate addresses updated to %q", newServerAddrs)
	return nil
}

// updateRequired returns true and a list of merged addresses if any of the
// new addresses are not yet contained in the server cert SAN list.
func updateRequired(serverCert string, newAddrs []string) ([]string, bool, error) {
	x509Cert, err := cert.ParseCert(serverCert)
	if err != nil {
		return nil, false, errors.Annotate(err, "cannot parse existing TLS certificate")
	}
	existingAddr := set.NewStrings()
	for _, ip := range x509Cert.IPAddresses {
		existingAddr.Add(ip.String())
	}
	logger.Debugf("existing cert addresses %v", existingAddr)
	logger.Debugf("new addresses %v", newAddrs)
	// Does newAddr contain any that are not already in existingAddr?
	newAddrSet := set.NewStrings(newAddrs...)
	update := newAddrSet.Difference(existingAddr).Size() > 0
	newAddrSet = newAddrSet.Union(existingAddr)
	return newAddrSet.SortedValues(), update, nil
}

// TearDown is defined on the NotifyWatchHandler interface.
func (c *CertificateUpdater) TearDown() error {
	select {
	case <-c.certChanged:
		// already closed
	default:
		close(c.certChanged)
	}
	return nil
}
<commit_msg>Do not close the channel in CertificateUpdater.TearDown.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package certupdater

import (
	"reflect"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/utils/set"

	"github.com/juju/juju/api/watcher"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/cert"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/network"
	"github.com/juju/juju/state"
	"github.com/juju/juju/worker"
)

var logger = loggo.GetLogger("juju.worker.certupdater")

// CertificateUpdater is responsible for generating state server certificates.
//
// In practice, CertificateUpdater is used by a state server's machine agent to watch
// that server's machines addresses in state, and write a new certificate to the
// agent's config file.
type CertificateUpdater struct {
	addressWatcher  AddressWatcher
	getter          StateServingInfoGetter
	setter          StateServingInfoSetter
	configGetter    EnvironConfigGetter
	hostPortsGetter APIHostPortsGetter
	certChanged     chan params.StateServingInfo
	addresses       []network.Address
}

// AddressWatcher is an interface that is provided to NewCertificateUpdater
// which can be used to watch for machine address changes.
type AddressWatcher interface {
	WatchAddresses() state.NotifyWatcher
	Addresses() (addresses []network.Address)
}

// EnvironConfigGetter is an interface that is provided to NewCertificateUpdater
// which can be used to get environment config.
type EnvironConfigGetter interface {
	EnvironConfig() (*config.Config, error)
}

// StateServingInfoGetter is an interface that is provided to NewCertificateUpdater
// whose StateServingInfo method will be invoked to get state serving info.
type StateServingInfoGetter interface {
	StateServingInfo() (params.StateServingInfo, bool)
}

// StateServingInfoSetter defines a function that is called to set a
// StateServingInfo value with a newly generated certificate.
type StateServingInfoSetter func(info params.StateServingInfo, done <-chan struct{}) error

// APIHostPortsGetter is an interface that is provided to NewCertificateUpdater
// whose APIHostPorts method will be invoked to get state server addresses.
type APIHostPortsGetter interface {
	APIHostPorts() ([][]network.HostPort, error)
}

// NewCertificateUpdater returns a worker.Worker that watches for changes to
// machine addresses and then generates a new state server certificate with those
// addresses in the certificate's SAN value.
func NewCertificateUpdater(addressWatcher AddressWatcher, getter StateServingInfoGetter,
	configGetter EnvironConfigGetter, hostPortsGetter APIHostPortsGetter, setter StateServingInfoSetter,
	certChanged chan params.StateServingInfo,
) worker.Worker {
	return worker.NewNotifyWorker(&CertificateUpdater{
		addressWatcher:  addressWatcher,
		configGetter:    configGetter,
		hostPortsGetter: hostPortsGetter,
		getter:          getter,
		setter:          setter,
		certChanged:     certChanged,
	})
}

// SetUp is defined on the NotifyWatchHandler interface.
func (c *CertificateUpdater) SetUp() (watcher.NotifyWatcher, error) {
	// Populate certificate SAN with any addresses we know about now.
	apiHostPorts, err := c.hostPortsGetter.APIHostPorts()
	if err != nil {
		return nil, errors.Annotate(err, "retrieving initial server addesses")
	}
	var initialSANAddresses []network.Address
	for _, server := range apiHostPorts {
		for _, nhp := range server {
			if nhp.Scope != network.ScopeCloudLocal {
				continue
			}
			initialSANAddresses = append(initialSANAddresses, nhp.Address)
		}
	}
	if err := c.updateCertificate(initialSANAddresses, make(chan struct{})); err != nil {
		return nil, errors.Annotate(err, "setting initial cerificate SAN list")
	}
	// Return
	return c.addressWatcher.WatchAddresses(), nil
}

// Handle is defined on the NotifyWatchHandler interface.
func (c *CertificateUpdater) Handle(done <-chan struct{}) error {
	addresses := c.addressWatcher.Addresses()
	if reflect.DeepEqual(addresses, c.addresses) {
		// Sometimes the watcher will tell us things have changed, when they
		// haven't as far as we can tell.
		logger.Debugf("addresses haven't really changed since last updated cert")
		return nil
	}
	return c.updateCertificate(addresses, done)
}

func (c *CertificateUpdater) updateCertificate(addresses []network.Address, done <-chan struct{}) error {
	logger.Debugf("new machine addresses: %#v", addresses)
	c.addresses = addresses

	// Older Juju deployments will not have the CA cert private key
	// available.
	stateInfo, ok := c.getter.StateServingInfo()
	if !ok {
		logger.Warningf("no state serving info, cannot regenerate server certificate")
		return nil
	}
	caPrivateKey := stateInfo.CAPrivateKey
	if caPrivateKey == "" {
		logger.Warningf("no CA cert private key, cannot regenerate server certificate")
		return nil
	}
	// Grab the env config and update a copy with ca cert private key.
	envConfig, err := c.configGetter.EnvironConfig()
	if err != nil {
		return errors.Annotate(err, "cannot read environment config")
	}
	envConfig, err = envConfig.Apply(map[string]interface{}{"ca-private-key": caPrivateKey})
	if err != nil {
		return errors.Annotate(err, "cannot add CA private key to environment config")
	}

	// For backwards compatibility, we must include "anything", "juju-apiserver"
	// and "juju-mongodb" as hostnames as that is what clients specify
	// as the hostname for verification (this certicate is used both
	// for serving MongoDB and API server connections).  We also
	// explicitly include localhost.
	serverAddrs := []string{"localhost", "juju-apiserver", "juju-mongodb", "anything"}
	for _, addr := range addresses {
		if addr.Value == "localhost" {
			continue
		}
		serverAddrs = append(serverAddrs, addr.Value)
	}
	newServerAddrs, update, err := updateRequired(stateInfo.Cert, serverAddrs)
	if err != nil {
		return errors.Annotate(err, "cannot determine if cert update needed")
	}
	if !update {
		logger.Debugf("no certificate update required")
		return nil
	}

	// Generate a new state server certificate with the machine addresses in the SAN value.
	newCert, newKey, err := envConfig.GenerateStateServerCertAndKey(newServerAddrs)
	if err != nil {
		return errors.Annotate(err, "cannot generate state server certificate")
	}
	stateInfo.Cert = string(newCert)
	stateInfo.PrivateKey = string(newKey)
	err = c.setter(stateInfo, done)
	if err != nil {
		return errors.Annotate(err, "cannot write agent config")
	}
	logger.Infof("State Server cerificate addresses updated to %q", newServerAddrs)
	return nil
}

// updateRequired returns true and a list of merged addresses if any of the
// new addresses are not yet contained in the server cert SAN list.
func updateRequired(serverCert string, newAddrs []string) ([]string, bool, error) {
	x509Cert, err := cert.ParseCert(serverCert)
	if err != nil {
		return nil, false, errors.Annotate(err, "cannot parse existing TLS certificate")
	}
	existingAddr := set.NewStrings()
	for _, ip := range x509Cert.IPAddresses {
		existingAddr.Add(ip.String())
	}
	logger.Debugf("existing cert addresses %v", existingAddr)
	logger.Debugf("new addresses %v", newAddrs)
	// Does newAddr contain any that are not already in existingAddr?
	newAddrSet := set.NewStrings(newAddrs...)
	update := newAddrSet.Difference(existingAddr).Size() > 0
	newAddrSet = newAddrSet.Union(existingAddr)
	return newAddrSet.SortedValues(), update, nil
}

// TearDown is defined on the NotifyWatchHandler interface.
func (c *CertificateUpdater) TearDown() error {
	// (lp:1493123) We do *not* close c.certChanged here because a call
	// to Handle() after TearDown() would result in a panic. That
	// sequence of calls happens when CertificateUpdater is wrapped in
	// NotifyWorker, that worker errors out and gets restarted by
	// a runner. In effect, CertificateUpdater doesn't properly "own"
	// the channel (c.certChanged). If it were refactored to correct
	// that then closing the channel here would be appropriate under
	// some conditions, like when NotifyWorker.Kill is called.
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The go-qemu Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package qmptest

import (
	"encoding/json"

	"github.com/digitalocean/go-qemu/qmp"
)

// A RunFunc is a function invoked to process a qmp.Command.  Any data
// returned by results is marshaled into JSON and returned to the caller.
type RunFunc func(cmd qmp.Command) (results interface{}, err error)

// NewMonitor creates a qmp.Monitor that invokes runFunc whenever its Run method
// is called.  All other methods are a no-op, and return nil.
func NewMonitor(runFunc RunFunc) qmp.Monitor {
	return &monitor{
		fn: runFunc,
	}
}

var _ qmp.Monitor = &monitor{}

type monitor struct {
	fn RunFunc
	noopMonitor
}

func (t *monitor) Run(raw []byte) ([]byte, error) {
	var cmd qmp.Command
	if err := json.Unmarshal(raw, &cmd); err != nil {
		return nil, err
	}

	result, err := t.fn(cmd)
	if err != nil {
		return nil, err
	}

	return json.Marshal(result)
}

var _ qmp.Monitor = &noopMonitor{}

type noopMonitor struct{}

func (noopMonitor) Connect() error                    { return nil }
func (noopMonitor) Disconnect() error                 { return nil }
func (noopMonitor) Run(_ []byte) ([]byte, error)      { return nil, nil }
func (noopMonitor) Events() (<-chan qmp.Event, error) { return nil, nil }
<commit_msg>qmp/qmptest: add a package comment<commit_after>// Copyright 2016 The go-qemu Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package qmptest provides types which assist in testing interactions with
// package qmp.
package qmptest

import (
	"encoding/json"

	"github.com/digitalocean/go-qemu/qmp"
)

// A RunFunc is a function invoked to process a qmp.Command.  Any data
// returned by results is marshaled into JSON and returned to the caller.
type RunFunc func(cmd qmp.Command) (results interface{}, err error)

// NewMonitor creates a qmp.Monitor that invokes runFunc whenever its Run method
// is called.  All other methods are a no-op, and return nil.
func NewMonitor(runFunc RunFunc) qmp.Monitor {
	return &monitor{
		fn: runFunc,
	}
}

var _ qmp.Monitor = &monitor{}

type monitor struct {
	fn RunFunc
	noopMonitor
}

func (t *monitor) Run(raw []byte) ([]byte, error) {
	var cmd qmp.Command
	if err := json.Unmarshal(raw, &cmd); err != nil {
		return nil, err
	}

	result, err := t.fn(cmd)
	if err != nil {
		return nil, err
	}

	return json.Marshal(result)
}

var _ qmp.Monitor = &noopMonitor{}

type noopMonitor struct{}

func (noopMonitor) Connect() error                    { return nil }
func (noopMonitor) Disconnect() error                 { return nil }
func (noopMonitor) Run(_ []byte) ([]byte, error)      { return nil, nil }
func (noopMonitor) Events() (<-chan qmp.Event, error) { return nil, nil }
<|endoftext|>


<commit_before>package filter

import (
	"cache"
	"config"
	"core"
	"logger"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"sync"
	"sync/atomic"
	"time"
	"utils"
	"io/ioutil"
	"strings"
	"os"
	"github.com/golang/groupcache/lru"
	"strconv"
)

const (
	DDOS_JS_REDIRECT = 10
	DDOS_JS_TIMEOUT  = 11
	DDOS_FLASH       = 2
	DDOS_CODE        = 3
)

type DdosFilter map[string]*DdosFilterThrottler

type DdosFilterThrottler struct {
	count        int64
	status       bool
	check_ticker *time.Ticker
	valid_ticker *time.Ticker
	tickerM      *sync.RWMutex
}

var _ core.RequestFilter = new(DdosFilter)
var lruCache = lru.New(5000)

func NewDdosFilter() (df DdosFilter) {
	df = make(DdosFilter)
	return
}
func (df DdosFilter) FilterRequest(request *core.Request) *http.Response {
	req := request.HttpRequest
	vhost := request.Context["config"].(*config.Vhost)

	if vhost.Ddos.Rtime == 0 || vhost.Ddos.Request == 0 {
		return nil
	}
	vhostname := vhost.Name

	//logger.Debug("url=%s %s r=%d rt=%d m=%d st=%d", req.URL.String(), vhostname, vhost.Ddos.Request, vhost.Ddos.Rtime, vhost.Ddos.Mode, vhost.Ddos.Stime)

	if _, ok := df[vhostname]; !ok {
		df[vhostname] = new(DdosFilterThrottler)
		df[vhostname].count = 0
		df[vhostname].status = false
		df[vhostname].check_ticker = time.NewTicker(time.Second * time.Duration(vhost.Ddos.Rtime))
		df[vhostname].valid_ticker = time.NewTicker(time.Second * time.Duration(vhost.Ddos.Stime))
		df[vhostname].tickerM = new(sync.RWMutex)
	}
	df[vhostname].tickerM.RLock()
	ct := df[vhostname].check_ticker
	vt := df[vhostname].valid_ticker
	df[vhostname].tickerM.RUnlock()

	if vt != nil && df[vhostname].status {
		RemoteAddr := request.RemoteAddr.String()
		ip, _, _ := net.SplitHostPort(RemoteAddr)

		//varify code
		if req.URL.Path == "/anti-ddos/code.png" && vhost.Ddos.Mode == DDOS_CODE {
			kfrom := ip + req.UserAgent();
			sessionkey := utils.Sha256(kfrom);
			codeindex := utils.RandomInt(1, 10);
			data, _ := df.getDdosCaptcha(sessionkey, codeindex);
			h := make(http.Header)
			h.Set("Content-Type", "image/png")
			h.Set("Pragma", "No-cache")
			return core.ByteResponse(request.HttpRequest, 200, h, data)
		}
		ckey := "ddos:" + vhostname + ":" + ip
		var cval string
		cval1 := cache.Get(ckey)
		if cval1 == nil {
			cval = utils.RandomString(utils.RandomInt(3, 7))
			cache.Put(ckey, cval, 5)

		} else {
			cval = cval1.(string)
			if cval == "pass" {
				return nil
			}
		}
		if vhost.Ddos.Mode == DDOS_CODE {
			kfrom := ip + req.UserAgent();
			key := utils.Sha256(kfrom);
			logger.Info("key= %s", key)
			if val, ok := lruCache.Get(key); ok {
				cval =  val.(string)
				logger.Info("captcha cval= %s", cval)
			}

		}
		isjoin := df.isJoinToWhitelist(req.URL, cval)
		ikey := "ccbl:" + vhostname + ":" + ip
		if isjoin {
			cache.Delete(ikey)
			cache.Put(ckey, "pass", 86400)
			return nil
		} else {
			if cache.IsExist(ikey) == false {
				cache.Put(ikey, 1, 600)
			} else {
				cache.Incr(ikey)
				hits := cache.Get(ikey).(int)
				if hits >= vhost.Ddos.Hits {
					logger.Info("block ip [%s] from %s", ip, vhostname)
					go func() {
						err := utils.AddToBlock(ip, vhost.Ddos.BlockTime)
						if err != nil {
							logger.Warn("block ip fail %s", err)
						}
					}()
				}
			}
		}
		request.Status |= STATUS_DDOS
		response := df.getDdosBody(req.URL.String(), cval, vhost.Ddos.Mode)
		return core.StringResponse(request.HttpRequest, 200, nil, response)
	}
	if ct != nil && request.Context["whitelist"] == false {
		atomic.AddInt64(&df[vhostname].count, 1)

		go func() {
			for {
				select {
				case <-ct.C:
					rps := atomic.LoadInt64(&df[vhostname].count)
					atomic.StoreInt64(&df[vhostname].count, 0)
					//core.Debug("%s RPS: %d", vhostname, atomic.LoadInt64(&df[vhostname].count))
					if rps >= vhost.Ddos.Request {
						df[vhostname].status = true
					}
				case <-vt.C:
					df[vhostname].status = false
				}
			}
			//atomic.AddInt64(&df[vhostname].count, -1)
		}()
	}
	return nil
}

func (df DdosFilter) getDdosCaptcha(key string, index int) ([]byte, error) {
	var err error

	code := config.GetDdosCaptcha()[index];
	lruCache.Add(key, code);

	if file, err := os.Open(df.buildFileName(index)); err == nil {
		defer file.Close()
		if data, err := ioutil.ReadAll(file); err == nil {

			return data, err
		} //if
	} //if
	return []byte{}, err
}

func (df DdosFilter) buildFileName(index int) string {
	return "captcha/" + strconv.Itoa(index) + ".png"
}

func (df DdosFilter) getDdosBody(link string, key string, mode int32) (body string) {
	uri, _ := url.Parse(link)
	q := uri.Query()
	q.Set("_l1O0", key)
	uri.RawQuery = q.Encode()
	switch mode {
	case DDOS_JS_REDIRECT:
		body = `<html><script>window.top.location = "` + uri.RequestURI() + `";</script></html>`
	case DDOS_JS_TIMEOUT:
		r := rand.Intn(7) + 1
		rkey := utils.RandomString(r)
		a := utils.Base64_encode(utils.Crypt(uri.RequestURI(), rkey))
		body = `<html></html>`
	case DDOS_CODE:
		body = `
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />

检测到您访问的网站正在遭受攻击，已经启动云盾防护机制。
请不用担心，输入验证码即可正常访问。给您带来不便，深表歉意。

<form method="get" action="" style="text-align:center">验证码：<input type="text" name="_l1O0" class="inputclass" /><input type="submit" class="button" value="提交" /></form>

`
	default:
		body = "the site was been attacked!"
	}
	return
}

func (df DdosFilter) isJoinToWhitelist(uri *url.URL, key string) bool {
	q := uri.Query()
	qkey := q.Get("_l1O0")
	if qkey != "" && strings.ToUpper(qkey) == strings.ToUpper(key) {
		return true
	}
	return false
}
<commit_msg>add resource<commit_after>package filter

import (
	"cache"
	"config"
	"core"
	"logger"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"sync"
	"sync/atomic"
	"time"
	"utils"
	"io/ioutil"
	"strings"
	"os"
	"github.com/golang/groupcache/lru"
	"strconv"
)

/*
 * http://th.atguy.com/mycode/xor_js_encryption/
 * https://gist.github.com/kapravel/9238281
 *
 */
const (
	DDOS_JS_REDIRECT = 10
	DDOS_JS_TIMEOUT  = 11
	DDOS_FLASH       = 2
	DDOS_CODE        = 3
)

type DdosFilter map[string]*DdosFilterThrottler

type DdosFilterThrottler struct {
	count        int64
	status       bool
	check_ticker *time.Ticker
	valid_ticker *time.Ticker
	tickerM      *sync.RWMutex
}

var _ core.RequestFilter = new(DdosFilter)
var lruCache = lru.New(5000)

func NewDdosFilter() (df DdosFilter) {
	df = make(DdosFilter)
	return
}
func (df DdosFilter) FilterRequest(request *core.Request) *http.Response {
	req := request.HttpRequest
	vhost := request.Context["config"].(*config.Vhost)

	if vhost.Ddos.Rtime == 0 || vhost.Ddos.Request == 0 {
		return nil
	}
	vhostname := vhost.Name

	//logger.Debug("url=%s %s r=%d rt=%d m=%d st=%d", req.URL.String(), vhostname, vhost.Ddos.Request, vhost.Ddos.Rtime, vhost.Ddos.Mode, vhost.Ddos.Stime)

	if _, ok := df[vhostname]; !ok {
		df[vhostname] = new(DdosFilterThrottler)
		df[vhostname].count = 0
		df[vhostname].status = false
		df[vhostname].check_ticker = time.NewTicker(time.Second * time.Duration(vhost.Ddos.Rtime))
		df[vhostname].valid_ticker = time.NewTicker(time.Second * time.Duration(vhost.Ddos.Stime))
		df[vhostname].tickerM = new(sync.RWMutex)
	}
	df[vhostname].tickerM.RLock()
	ct := df[vhostname].check_ticker
	vt := df[vhostname].valid_ticker
	df[vhostname].tickerM.RUnlock()

	if vt != nil && df[vhostname].status {
		RemoteAddr := request.RemoteAddr.String()
		ip, _, _ := net.SplitHostPort(RemoteAddr)

		//varify code
		if req.URL.Path == "/anti-ddos/code.png" && vhost.Ddos.Mode == DDOS_CODE {
			kfrom := ip + req.UserAgent();
			sessionkey := utils.Sha256(kfrom);
			codeindex := utils.RandomInt(1, 10);
			data, _ := df.getDdosCaptcha(sessionkey, codeindex);
			h := make(http.Header)
			h.Set("Content-Type", "image/png")
			h.Set("Pragma", "No-cache")
			return core.ByteResponse(request.HttpRequest, 200, h, data)
		}
		ckey := "ddos:" + vhostname + ":" + ip
		var cval string
		cval1 := cache.Get(ckey)
		if cval1 == nil {
			cval = utils.RandomString(utils.RandomInt(3, 7))
			cache.Put(ckey, cval, 5)

		} else {
			cval = cval1.(string)
			if cval == "pass" {
				return nil
			}
		}
		if vhost.Ddos.Mode == DDOS_CODE {
			kfrom := ip + req.UserAgent();
			key := utils.Sha256(kfrom);
			logger.Info("key= %s", key)
			if val, ok := lruCache.Get(key); ok {
				cval =  val.(string)
				logger.Info("captcha cval= %s", cval)
			}

		}
		isjoin := df.isJoinToWhitelist(req.URL, cval)
		ikey := "ccbl:" + vhostname + ":" + ip
		if isjoin {
			cache.Delete(ikey)
			cache.Put(ckey, "pass", 86400)
			return nil
		} else {
			if cache.IsExist(ikey) == false {
				cache.Put(ikey, 1, 600)
			} else {
				cache.Incr(ikey)
				hits := cache.Get(ikey).(int)
				if hits >= vhost.Ddos.Hits {
					logger.Info("block ip [%s] from %s", ip, vhostname)
					go func() {
						err := utils.AddToBlock(ip, vhost.Ddos.BlockTime)
						if err != nil {
							logger.Warn("block ip fail %s", err)
						}
					}()
				}
			}
		}
		request.Status |= STATUS_DDOS
		response := df.getDdosBody(req.URL.String(), cval, vhost.Ddos.Mode)
		return core.StringResponse(request.HttpRequest, 200, nil, response)
	}
	if ct != nil && request.Context["whitelist"] == false {
		atomic.AddInt64(&df[vhostname].count, 1)

		go func() {
			for {
				select {
				case <-ct.C:
					rps := atomic.LoadInt64(&df[vhostname].count)
					atomic.StoreInt64(&df[vhostname].count, 0)
					//core.Debug("%s RPS: %d", vhostname, atomic.LoadInt64(&df[vhostname].count))
					if rps >= vhost.Ddos.Request {
						df[vhostname].status = true
					}
				case <-vt.C:
					df[vhostname].status = false
				}
			}
			//atomic.AddInt64(&df[vhostname].count, -1)
		}()
	}
	return nil
}

func (df DdosFilter) getDdosCaptcha(key string, index int) ([]byte, error) {
	var err error

	code := config.GetDdosCaptcha()[index];
	lruCache.Add(key, code);

	if file, err := os.Open(df.buildFileName(index)); err == nil {
		defer file.Close()
		if data, err := ioutil.ReadAll(file); err == nil {

			return data, err
		} //if
	} //if
	return []byte{}, err
}

func (df DdosFilter) buildFileName(index int) string {
	return "captcha/" + strconv.Itoa(index) + ".png"
}

func (df DdosFilter) getDdosBody(link string, key string, mode int32) (body string) {
	uri, _ := url.Parse(link)
	q := uri.Query()
	q.Set("_l1O0", key)
	uri.RawQuery = q.Encode()
	switch mode {
	case DDOS_JS_REDIRECT:
		body = `<html><script>window.top.location = "` + uri.RequestURI() + `";</script></html>`
	case DDOS_JS_TIMEOUT:
		r := rand.Intn(7) + 1
		rkey := utils.RandomString(r)
		a := utils.Base64_encode(utils.Crypt(uri.RequestURI(), rkey))
		body = `<html></html>`
	case DDOS_CODE:
		body = `
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />

检测到您访问的网站正在遭受攻击，已经启动云盾防护机制。
请不用担心，输入验证码即可正常访问。给您带来不便，深表歉意。

<form method="get" action="" style="text-align:center">验证码：<input type="text" name="_l1O0" class="inputclass" /><input type="submit" class="button" value="提交" /></form>

`
	default:
		body = "the site was been attacked!"
	}
	return
}

func (df DdosFilter) isJoinToWhitelist(uri *url.URL, key string) bool {
	q := uri.Query()
	qkey := q.Get("_l1O0")
	if qkey != "" && strings.ToUpper(qkey) == strings.ToUpper(key) {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package doc extracts source code documentation from a Go AST.
package doc

import (
	"go/ast"
	"go/token"
)

// Package is the documentation for an entire package.
type Package struct {
	Doc        string
	Name       string
	ImportPath string
	Imports    []string
	Filenames  []string
	Bugs       []string

	// declarations
	Consts []*Value
	Types  []*Type
	Vars   []*Value
	Funcs  []*Func
}

// Value is the documentation for a (possibly grouped) var or const declaration.
type Value struct {
	Doc   string
	Names []string // var or const names in declaration order
	Decl  *ast.GenDecl

	order int
}

// Type is the documentation for type declaration.
type Type struct {
	Doc  string
	Name string
	Decl *ast.GenDecl

	// associated declarations
	Consts  []*Value // sorted list of constants of (mostly) this type
	Vars    []*Value // sorted list of variables of (mostly) this type
	Funcs   []*Func  // sorted list of functions returning this type
	Methods []*Func  // sorted list of methods (including embedded ones) of this type
}

// Func is the documentation for a func declaration.
type Func struct {
	Doc  string
	Name string
	Decl *ast.FuncDecl

	// methods
	// (for functions, these fields have the respective zero value)
	Recv  string // actual   receiver "T" or "*T"
	Orig  string // original receiver "T" or "*T"
	Level int    // embedding level; 0 means not embedded
}

// Mode values control the operation of New.
type Mode int

const (
	// extract documentation for all package-level declarations,
	// not just exported ones
	AllDecls Mode = 1 << iota
)

// New computes the package documentation for the given package AST.
// New takes ownership of the AST pkg and may edit or overwrite it.
//
func New(pkg *ast.Package, importPath string, mode Mode) *Package {
	var r reader
	r.readPackage(pkg, mode)
	r.computeMethodSets()
	r.cleanupTypes()
	return &Package{
		Doc:        r.doc,
		Name:       pkg.Name,
		ImportPath: importPath,
		Imports:    sortedKeys(r.imports),
		Filenames:  r.filenames,
		Bugs:       r.bugs,
		Consts:     sortedValues(r.values, token.CONST),
		Types:      sortedTypes(r.types),
		Vars:       sortedValues(r.values, token.VAR),
		Funcs:      sortedFuncs(r.funcs),
	}
}
<commit_msg>go/doc: fix typo<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package doc extracts source code documentation from a Go AST.
package doc

import (
	"go/ast"
	"go/token"
)

// Package is the documentation for an entire package.
type Package struct {
	Doc        string
	Name       string
	ImportPath string
	Imports    []string
	Filenames  []string
	Bugs       []string

	// declarations
	Consts []*Value
	Types  []*Type
	Vars   []*Value
	Funcs  []*Func
}

// Value is the documentation for a (possibly grouped) var or const declaration.
type Value struct {
	Doc   string
	Names []string // var or const names in declaration order
	Decl  *ast.GenDecl

	order int
}

// Type is the documentation for a type declaration.
type Type struct {
	Doc  string
	Name string
	Decl *ast.GenDecl

	// associated declarations
	Consts  []*Value // sorted list of constants of (mostly) this type
	Vars    []*Value // sorted list of variables of (mostly) this type
	Funcs   []*Func  // sorted list of functions returning this type
	Methods []*Func  // sorted list of methods (including embedded ones) of this type
}

// Func is the documentation for a func declaration.
type Func struct {
	Doc  string
	Name string
	Decl *ast.FuncDecl

	// methods
	// (for functions, these fields have the respective zero value)
	Recv  string // actual   receiver "T" or "*T"
	Orig  string // original receiver "T" or "*T"
	Level int    // embedding level; 0 means not embedded
}

// Mode values control the operation of New.
type Mode int

const (
	// extract documentation for all package-level declarations,
	// not just exported ones
	AllDecls Mode = 1 << iota
)

// New computes the package documentation for the given package AST.
// New takes ownership of the AST pkg and may edit or overwrite it.
//
func New(pkg *ast.Package, importPath string, mode Mode) *Package {
	var r reader
	r.readPackage(pkg, mode)
	r.computeMethodSets()
	r.cleanupTypes()
	return &Package{
		Doc:        r.doc,
		Name:       pkg.Name,
		ImportPath: importPath,
		Imports:    sortedKeys(r.imports),
		Filenames:  r.filenames,
		Bugs:       r.bugs,
		Consts:     sortedValues(r.values, token.CONST),
		Types:      sortedTypes(r.types),
		Vars:       sortedValues(r.values, token.VAR),
		Funcs:      sortedFuncs(r.funcs),
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The math package provides basic constants and mathematical functions.
package math

// Mathematical constants.
// Reference: http://www.research.att.com/~njas/sequences/Axxxxxx
const (
	E	= 2.71828182845904523536028747135266249775724709369995957496696763;  // A001113
	Pi	= 3.14159265358979323846264338327950288419716939937510582097494459;  // A000796
	Phi	= 1.61803398874989484820458683436563811772030917980576286213544862;  // A001622

	Sqrt2	= 1.41421356237309504880168872420969807856967187537694807317667974;  // A002193
	SqrtE	= 1.64872127070012814684865078781416357165377610071014801157507931;  // A019774
	SqrtPi	= 1.77245385090551602729816748334114518279754945612238712821380779;  // A002161
	SqrtPhi	= 1.27201964951406896425242246173749149171560804184009624861664038;  // A139339

	Ln2	= 0.693147180559945309417232121458176568075500134360255254120680009; // A002162
	Log2E	= 1/Ln2;
	Ln10	= 2.30258509299404568401799145468436420760110148862877297603332790;  // A002392
	Log10E	= 1/Ln10;
)

// BUG(rsc): The manual should define the special cases for all of these functions.
<commit_msg>constants for floating point limits<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The math package provides basic constants and mathematical functions.
package math

// Mathematical constants.
// Reference: http://www.research.att.com/~njas/sequences/Axxxxxx
const (
	E	= 2.71828182845904523536028747135266249775724709369995957496696763;  // A001113
	Pi	= 3.14159265358979323846264338327950288419716939937510582097494459;  // A000796
	Phi	= 1.61803398874989484820458683436563811772030917980576286213544862;  // A001622

	Sqrt2	= 1.41421356237309504880168872420969807856967187537694807317667974;  // A002193
	SqrtE	= 1.64872127070012814684865078781416357165377610071014801157507931;  // A019774
	SqrtPi	= 1.77245385090551602729816748334114518279754945612238712821380779;  // A002161
	SqrtPhi	= 1.27201964951406896425242246173749149171560804184009624861664038;  // A139339

	Ln2	= 0.693147180559945309417232121458176568075500134360255254120680009; // A002162
	Log2E	= 1/Ln2;
	Ln10	= 2.30258509299404568401799145468436420760110148862877297603332790;  // A002392
	Log10E	= 1/Ln10;

	MaxFloat32	= 3.40282346638528860e+38;
	MinFloat32	= 1.40129846432481707e-45;
	MaxFloat64	= 1.7976931348623157e+308;
	MinFloat64	= 5.0e-324;
)

// BUG(rsc): The manual should define the special cases for all of these functions.
<|endoftext|>


<commit_before>// package stickysession is a mixin for load balancers that implements layer 8 (http cookie) session affinity
package roundrobin

import (
	"net/http"
	"net/url"
)

type StickySession struct {
	cookiename string
}

func NewStickySession(c string) *StickySession {
	return &StickySession{c}
}

// GetBackend returns the backend URL stored in the sticky cookie, iff the backend is still in the valid list of servers.
func (s *StickySession) GetBackend(req *http.Request, servers []*url.URL) (*url.URL, bool, error) {
	cookie, err := req.Cookie(s.cookiename)
	switch err {
	case nil:
	case http.ErrNoCookie:
		return nil, false, nil
	default:
		return nil, false, err
	}

	s_url, err := url.Parse(cookie.Value)
	if err != nil {
		return nil, false, err
	}

	if s.isBackendAlive(s_url, servers) {
		return s_url, true, nil
	} else {
		return nil, false, nil
	}
}

func (s *StickySession) StickBackend(backend *url.URL, w *http.ResponseWriter) {
	c := &http.Cookie{Name: s.cookiename, Value: backend.String()}
	http.SetCookie(*w, c)
	return
}

func (s *StickySession) isBackendAlive(needle *url.URL, haystack []*url.URL) bool {
	if len(haystack) == 0 {
		return false
	}

	for _, s := range haystack {
		if sameURL(needle, s) {
			return true
		}
	}
	return false
}
<commit_msg>fix misleading comment<commit_after>// package stickysession is a mixin for load balancers that implements layer 7 (http cookie) session affinity
package roundrobin

import (
	"net/http"
	"net/url"
)

type StickySession struct {
	cookiename string
}

func NewStickySession(c string) *StickySession {
	return &StickySession{c}
}

// GetBackend returns the backend URL stored in the sticky cookie, iff the backend is still in the valid list of servers.
func (s *StickySession) GetBackend(req *http.Request, servers []*url.URL) (*url.URL, bool, error) {
	cookie, err := req.Cookie(s.cookiename)
	switch err {
	case nil:
	case http.ErrNoCookie:
		return nil, false, nil
	default:
		return nil, false, err
	}

	s_url, err := url.Parse(cookie.Value)
	if err != nil {
		return nil, false, err
	}

	if s.isBackendAlive(s_url, servers) {
		return s_url, true, nil
	} else {
		return nil, false, nil
	}
}

func (s *StickySession) StickBackend(backend *url.URL, w *http.ResponseWriter) {
	c := &http.Cookie{Name: s.cookiename, Value: backend.String()}
	http.SetCookie(*w, c)
	return
}

func (s *StickySession) isBackendAlive(needle *url.URL, haystack []*url.URL) bool {
	if len(haystack) == 0 {
		return false
	}

	for _, s := range haystack {
		if sameURL(needle, s) {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package resolver

import (
	"crypto/tls"
	"crypto/x509"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/containerd/containerd/remotes/docker"
	"github.com/moby/buildkit/cmd/buildkitd/config"
	"github.com/moby/buildkit/util/tracing"
	"github.com/pkg/errors"
)

func fillInsecureOpts(host string, c config.RegistryConfig, h docker.RegistryHost) ([]docker.RegistryHost, error) {
	var hosts []docker.RegistryHost

	tc, err := loadTLSConfig(c)
	if err != nil {
		return nil, err
	}
	var isHTTP bool

	if c.PlainHTTP != nil && *c.PlainHTTP {
		isHTTP = true
	}
	if c.PlainHTTP == nil {
		if ok, _ := docker.MatchLocalhost(host); ok {
			isHTTP = true
		}
	}

	if isHTTP {
		h2 := h
		h2.Scheme = "http"
		hosts = append(hosts, h2)
	}
	if c.Insecure != nil && *c.Insecure {
		h2 := h
		transport := newDefaultTransport()
		transport.TLSClientConfig = tc
		h2.Client = &http.Client{
			Transport: tracing.NewTransport(transport),
		}
		tc.InsecureSkipVerify = true
		hosts = append(hosts, h2)
	}

	if len(hosts) == 0 {
		transport := newDefaultTransport()
		transport.TLSClientConfig = tc

		h.Client = &http.Client{
			Transport: tracing.NewTransport(transport),
		}
		hosts = append(hosts, h)
	}

	return hosts, nil
}

func loadTLSConfig(c config.RegistryConfig) (*tls.Config, error) {
	for _, d := range c.TLSConfigDir {
		fs, err := ioutil.ReadDir(d)
		if err != nil && !errors.Is(err, os.ErrNotExist) && !errors.Is(err, os.ErrPermission) {
			return nil, errors.WithStack(err)
		}
		for _, f := range fs {
			if strings.HasSuffix(f.Name(), ".crt") {
				c.RootCAs = append(c.RootCAs, filepath.Join(d, f.Name()))
			}
			if strings.HasSuffix(f.Name(), ".cert") {
				c.KeyPairs = append(c.KeyPairs, config.TLSKeyPair{
					Certificate: filepath.Join(d, f.Name()),
					Key:         filepath.Join(d, strings.TrimSuffix(f.Name(), ".cert")+".key"),
				})
			}
		}
	}

	tc := &tls.Config{}
	if len(c.RootCAs) > 0 {
		systemPool, err := x509.SystemCertPool()
		if err != nil {
			if runtime.GOOS == "windows" {
				systemPool = x509.NewCertPool()
			} else {
				return nil, errors.Wrapf(err, "unable to get system cert pool")
			}
		}
		tc.RootCAs = systemPool
	}

	for _, p := range c.RootCAs {
		dt, err := ioutil.ReadFile(p)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to read %s", p)
		}
		tc.RootCAs.AppendCertsFromPEM(dt)
	}

	for _, kp := range c.KeyPairs {
		cert, err := tls.LoadX509KeyPair(kp.Certificate, kp.Key)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to load keypair for %s", kp.Certificate)
		}
		tc.Certificates = append(tc.Certificates, cert)
	}
	return tc, nil
}

// NewRegistryConfig converts registry config to docker.RegistryHosts callback
func NewRegistryConfig(m map[string]config.RegistryConfig) docker.RegistryHosts {
	return docker.Registries(
		func(host string) ([]docker.RegistryHost, error) {
			c, ok := m[host]
			if !ok {
				return nil, nil
			}

			var out []docker.RegistryHost

			for _, mirror := range c.Mirrors {
				h := docker.RegistryHost{
					Scheme:       "https",
					Client:       newDefaultClient(),
					Host:         mirror,
					Path:         "/v2",
					Capabilities: docker.HostCapabilityPull | docker.HostCapabilityResolve,
				}

				hosts, err := fillInsecureOpts(mirror, m[mirror], h)
				if err != nil {
					return nil, err
				}

				out = append(out, hosts...)
			}

			if host == "docker.io" {
				host = "registry-1.docker.io"
			}

			h := docker.RegistryHost{
				Scheme:       "https",
				Client:       newDefaultClient(),
				Host:         host,
				Path:         "/v2",
				Capabilities: docker.HostCapabilityPush | docker.HostCapabilityPull | docker.HostCapabilityResolve,
			}

			hosts, err := fillInsecureOpts(host, c, h)
			if err != nil {
				return nil, err
			}

			out = append(out, hosts...)
			return out, nil
		},
		docker.ConfigureDefaultRegistries(
			docker.WithClient(newDefaultClient()),
			docker.WithPlainHTTP(docker.MatchLocalhost),
		),
	)
}

func newDefaultClient() *http.Client {
	return &http.Client{
		Transport: tracing.NewTransport(newDefaultTransport()),
	}
}

// newDefaultTransport is for pull or push client
//
// NOTE: For push, there must disable http2 for https because the flow control
// will limit data transfer. The net/http package doesn't provide http2 tunable
// settings which limits push performance.
//
// REF: https://github.com/golang/go/issues/14077
func newDefaultTransport() *http.Transport {
	return &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 60 * time.Second,
		}).DialContext,
		MaxIdleConns:          10,
		IdleConnTimeout:       30 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 5 * time.Second,
		TLSNextProto:          make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
	}
}
<commit_msg>resolver: fix tcp connections limit<commit_after>package resolver

import (
	"crypto/tls"
	"crypto/x509"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/containerd/containerd/remotes/docker"
	"github.com/moby/buildkit/cmd/buildkitd/config"
	"github.com/moby/buildkit/util/tracing"
	"github.com/pkg/errors"
)

func fillInsecureOpts(host string, c config.RegistryConfig, h docker.RegistryHost) ([]docker.RegistryHost, error) {
	var hosts []docker.RegistryHost

	tc, err := loadTLSConfig(c)
	if err != nil {
		return nil, err
	}
	var isHTTP bool

	if c.PlainHTTP != nil && *c.PlainHTTP {
		isHTTP = true
	}
	if c.PlainHTTP == nil {
		if ok, _ := docker.MatchLocalhost(host); ok {
			isHTTP = true
		}
	}

	if isHTTP {
		h2 := h
		h2.Scheme = "http"
		hosts = append(hosts, h2)
	}
	if c.Insecure != nil && *c.Insecure {
		h2 := h
		transport := newTransport()
		transport.TLSClientConfig = tc
		h2.Client = &http.Client{
			Transport: tracing.NewTransport(transport),
		}
		tc.InsecureSkipVerify = true
		hosts = append(hosts, h2)
	}

	if len(hosts) == 0 {
		transport := newTransport()
		transport.TLSClientConfig = tc

		h.Client = &http.Client{
			Transport: tracing.NewTransport(transport),
		}
		hosts = append(hosts, h)
	}

	return hosts, nil
}

func loadTLSConfig(c config.RegistryConfig) (*tls.Config, error) {
	for _, d := range c.TLSConfigDir {
		fs, err := ioutil.ReadDir(d)
		if err != nil && !errors.Is(err, os.ErrNotExist) && !errors.Is(err, os.ErrPermission) {
			return nil, errors.WithStack(err)
		}
		for _, f := range fs {
			if strings.HasSuffix(f.Name(), ".crt") {
				c.RootCAs = append(c.RootCAs, filepath.Join(d, f.Name()))
			}
			if strings.HasSuffix(f.Name(), ".cert") {
				c.KeyPairs = append(c.KeyPairs, config.TLSKeyPair{
					Certificate: filepath.Join(d, f.Name()),
					Key:         filepath.Join(d, strings.TrimSuffix(f.Name(), ".cert")+".key"),
				})
			}
		}
	}

	tc := &tls.Config{}
	if len(c.RootCAs) > 0 {
		systemPool, err := x509.SystemCertPool()
		if err != nil {
			if runtime.GOOS == "windows" {
				systemPool = x509.NewCertPool()
			} else {
				return nil, errors.Wrapf(err, "unable to get system cert pool")
			}
		}
		tc.RootCAs = systemPool
	}

	for _, p := range c.RootCAs {
		dt, err := ioutil.ReadFile(p)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to read %s", p)
		}
		tc.RootCAs.AppendCertsFromPEM(dt)
	}

	for _, kp := range c.KeyPairs {
		cert, err := tls.LoadX509KeyPair(kp.Certificate, kp.Key)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to load keypair for %s", kp.Certificate)
		}
		tc.Certificates = append(tc.Certificates, cert)
	}
	return tc, nil
}

// NewRegistryConfig converts registry config to docker.RegistryHosts callback
func NewRegistryConfig(m map[string]config.RegistryConfig) docker.RegistryHosts {
	return docker.Registries(
		func(host string) ([]docker.RegistryHost, error) {
			c, ok := m[host]
			if !ok {
				return nil, nil
			}

			var out []docker.RegistryHost

			for _, mirror := range c.Mirrors {
				h := docker.RegistryHost{
					Scheme:       "https",
					Client:       newDefaultClient(),
					Host:         mirror,
					Path:         "/v2",
					Capabilities: docker.HostCapabilityPull | docker.HostCapabilityResolve,
				}

				hosts, err := fillInsecureOpts(mirror, m[mirror], h)
				if err != nil {
					return nil, err
				}

				out = append(out, hosts...)
			}

			if host == "docker.io" {
				host = "registry-1.docker.io"
			}

			h := docker.RegistryHost{
				Scheme:       "https",
				Client:       newDefaultClient(),
				Host:         host,
				Path:         "/v2",
				Capabilities: docker.HostCapabilityPush | docker.HostCapabilityPull | docker.HostCapabilityResolve,
			}

			hosts, err := fillInsecureOpts(host, c, h)
			if err != nil {
				return nil, err
			}

			out = append(out, hosts...)
			return out, nil
		},
		docker.ConfigureDefaultRegistries(
			docker.WithClient(newDefaultClient()),
			docker.WithPlainHTTP(docker.MatchLocalhost),
		),
	)
}

func newDefaultClient() *http.Client {
	return &http.Client{
		Transport: tracing.NewTransport(defaultTransport),
	}
}

var defaultTransport = newTransport()

// newTransport is for pull or push client
//
// NOTE: For push, there must disable http2 for https because the flow control
// will limit data transfer. The net/http package doesn't provide http2 tunable
// settings which limits push performance.
//
// REF: https://github.com/golang/go/issues/14077
func newTransport() *http.Transport {
	return &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 60 * time.Second,
		}).DialContext,
		MaxIdleConns:          10,
		IdleConnTimeout:       30 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 5 * time.Second,
		MaxConnsPerHost:       6,
		TLSNextProto:          make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
	}
}
<|endoftext|>


<commit_before>package validate

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"testing"

	"github.com/hashicorp/go-multierror"
	rspec "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/stretchr/testify/assert"

	"github.com/opencontainers/runtime-tools/specerror"
)

func TestNewValidator(t *testing.T) {
	testSpec := &rspec.Spec{}
	testBundle := ""
	testPlatform := "not" + runtime.GOOS
	cases := []struct {
		val      Validator
		expected Validator
	}{
		{Validator{testSpec, testBundle, true, testPlatform}, Validator{testSpec, testBundle, true, runtime.GOOS}},
		{Validator{testSpec, testBundle, true, runtime.GOOS}, Validator{testSpec, testBundle, true, runtime.GOOS}},
		{Validator{testSpec, testBundle, false, testPlatform}, Validator{testSpec, testBundle, false, testPlatform}},
	}

	for _, c := range cases {
		assert.Equal(t, c.expected, NewValidator(c.val.spec, c.val.bundlePath, c.val.HostSpecific, c.val.platform))
	}
}

func TestJSONSchema(t *testing.T) {
	for _, tt := range []struct {
		config *rspec.Spec
		error  string
	}{
		{
			config: &rspec.Spec{},
			error:  "Version string empty",
		},
		{
			config: &rspec.Spec{
				Version: "1.0.1-rc1",
			},
			error: "Could not read schema from HTTP, response status is 404 Not Found",
		},
		{
			config: &rspec.Spec{
				Version: "1.0.0",
			},
			error: "",
		},
		{
			config: &rspec.Spec{
				Version: "1.0.0",
				Process: &rspec.Process{},
			},
			error: "process.args: Invalid type. Expected: array, given: null",
		},
		{
			config: &rspec.Spec{
				Version: "1.0.0-rc5",
			},
			error: "process: process is required",
		},
	} {
		t.Run(tt.error, func(t *testing.T) {
			v := &Validator{spec: tt.config}
			errs := v.CheckJSONSchema()
			if tt.error == "" {
				assert.Equal(t, nil, errs)
				return
			}
			merr, ok := errs.(*multierror.Error)
			if !ok {
				t.Fatalf("non-multierror returned by CheckJSONSchema: %s", errs.Error())
			}
			for _, err := range merr.Errors {
				if err.Error() == tt.error {
					return
				}
			}
			assert.Equal(t, tt.error, errs.Error())
		})
	}
}

func TestCheckRoot(t *testing.T) {
	tmpBundle, err := ioutil.TempDir("", "oci-check-rootfspath")
	if err != nil {
		t.Fatalf("Failed to create a TempDir in 'CheckRoot'")
	}
	defer os.RemoveAll(tmpBundle)

	rootfsDir := "rootfs/rootfs"
	rootfsNonDir := "rootfsfile"
	rootfsNonExists := "rootfsnil"
	if err := os.MkdirAll(filepath.Join(tmpBundle, rootfsDir), 0700); err != nil {
		t.Fatalf("Failed to create a rootfs directory in 'CheckRoot'")
	}
	if _, err := os.Create(filepath.Join(tmpBundle, rootfsNonDir)); err != nil {
		t.Fatalf("Failed to create a non-directory rootfs in 'CheckRoot'")
	}

	// Note: Abs error is not tested
	cases := []struct {
		val      rspec.Spec
		platform string
		expected specerror.Code
	}{
		{rspec.Spec{Windows: &rspec.Windows{HyperV: &rspec.WindowsHyperV{}}, Root: &rspec.Root{}}, "windows", specerror.RootOnHyperV},
		{rspec.Spec{Windows: &rspec.Windows{HyperV: &rspec.WindowsHyperV{}}, Root: nil}, "windows", specerror.NonError},
		{rspec.Spec{Root: nil}, "linux", specerror.RootOnNonHyperV},
		{rspec.Spec{Root: &rspec.Root{Path: "maverick-rootfs"}}, "linux", specerror.PathName},
		{rspec.Spec{Root: &rspec.Root{Path: "rootfs"}}, "linux", specerror.NonError},
		{rspec.Spec{Root: &rspec.Root{Path: filepath.Join(tmpBundle, rootfsNonExists)}}, "linux", specerror.PathExistence},
		{rspec.Spec{Root: &rspec.Root{Path: filepath.Join(tmpBundle, rootfsNonDir)}}, "linux", specerror.PathExistence},
		{rspec.Spec{Root: &rspec.Root{Path: filepath.Join(tmpBundle, "rootfs")}}, "linux", specerror.NonError},
		{rspec.Spec{Root: &rspec.Root{Path: "rootfs/rootfs"}}, "linux", specerror.ArtifactsInSingleDir},
		{rspec.Spec{Root: &rspec.Root{Readonly: true}}, "windows", specerror.ReadonlyOnWindows},
	}
	for _, c := range cases {
		v := NewValidator(&c.val, tmpBundle, false, c.platform)
		err := v.CheckRoot()
		assert.Equal(t, c.expected, specerror.FindError(err, c.expected), fmt.Sprintf("Fail to check Root: %v %d", err, c.expected))
	}
}

func TestCheckSemVer(t *testing.T) {
	cases := []struct {
		val      string
		expected specerror.Code
	}{
		{rspec.Version, specerror.NonError},
		//FIXME: validate currently only handles rpsec.Version
		{"0.0.1", specerror.NonRFCError},
		{"invalid", specerror.SpecVersion},
	}

	for _, c := range cases {
		v := NewValidator(&rspec.Spec{Version: c.val}, "", false, "linux")
		err := v.CheckSemVer()
		assert.Equal(t, c.expected, specerror.FindError(err, c.expected), "Fail to check SemVer "+c.val)
	}
}
<commit_msg>validate/validate_test: Add linux.rootfsPropagation checks<commit_after>package validate

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"testing"

	"github.com/hashicorp/go-multierror"
	rspec "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/stretchr/testify/assert"

	"github.com/opencontainers/runtime-tools/specerror"
)

func TestNewValidator(t *testing.T) {
	testSpec := &rspec.Spec{}
	testBundle := ""
	testPlatform := "not" + runtime.GOOS
	cases := []struct {
		val      Validator
		expected Validator
	}{
		{Validator{testSpec, testBundle, true, testPlatform}, Validator{testSpec, testBundle, true, runtime.GOOS}},
		{Validator{testSpec, testBundle, true, runtime.GOOS}, Validator{testSpec, testBundle, true, runtime.GOOS}},
		{Validator{testSpec, testBundle, false, testPlatform}, Validator{testSpec, testBundle, false, testPlatform}},
	}

	for _, c := range cases {
		assert.Equal(t, c.expected, NewValidator(c.val.spec, c.val.bundlePath, c.val.HostSpecific, c.val.platform))
	}
}

func TestJSONSchema(t *testing.T) {
	for _, tt := range []struct {
		config *rspec.Spec
		error  string
	}{
		{
			config: &rspec.Spec{},
			error:  "Version string empty",
		},
		{
			config: &rspec.Spec{
				Version: "1.0.1-rc1",
			},
			error: "Could not read schema from HTTP, response status is 404 Not Found",
		},
		{
			config: &rspec.Spec{
				Version: "1.0.0",
			},
			error: "",
		},
		{
			config: &rspec.Spec{
				Version: "1.0.0",
				Process: &rspec.Process{},
			},
			error: "process.args: Invalid type. Expected: array, given: null",
		},
		{
			config: &rspec.Spec{
				Version: "1.0.0",
				Linux:   &rspec.Linux{},
			},
			error: "",
		},
		{
			config: &rspec.Spec{
				Version: "1.0.0",
				Linux: &rspec.Linux{
					RootfsPropagation: "",
				},
			},
			error: "",
		},
		{
			config: &rspec.Spec{
				Version: "1.0.0",
				Linux: &rspec.Linux{
					RootfsPropagation: "shared",
				},
			},
			error: "",
		},
		{
			config: &rspec.Spec{
				Version: "1.0.0",
				Linux: &rspec.Linux{
					RootfsPropagation: "rshared",
				},
			},
			error: "linux.rootfsPropagation: linux.rootfsPropagation must be one of the following: \"private\", \"shared\", \"slave\", \"unbindable\"",
		},
		{
			config: &rspec.Spec{
				Version: "1.0.0-rc5",
			},
			error: "process: process is required",
		},
	} {
		t.Run(tt.error, func(t *testing.T) {
			v := &Validator{spec: tt.config}
			errs := v.CheckJSONSchema()
			if tt.error == "" {
				assert.Equal(t, nil, errs)
				return
			}
			merr, ok := errs.(*multierror.Error)
			if !ok {
				t.Fatalf("non-multierror returned by CheckJSONSchema: %s", errs.Error())
			}
			for _, err := range merr.Errors {
				if err.Error() == tt.error {
					return
				}
			}
			assert.Equal(t, tt.error, errs.Error())
		})
	}
}

func TestCheckRoot(t *testing.T) {
	tmpBundle, err := ioutil.TempDir("", "oci-check-rootfspath")
	if err != nil {
		t.Fatalf("Failed to create a TempDir in 'CheckRoot'")
	}
	defer os.RemoveAll(tmpBundle)

	rootfsDir := "rootfs/rootfs"
	rootfsNonDir := "rootfsfile"
	rootfsNonExists := "rootfsnil"
	if err := os.MkdirAll(filepath.Join(tmpBundle, rootfsDir), 0700); err != nil {
		t.Fatalf("Failed to create a rootfs directory in 'CheckRoot'")
	}
	if _, err := os.Create(filepath.Join(tmpBundle, rootfsNonDir)); err != nil {
		t.Fatalf("Failed to create a non-directory rootfs in 'CheckRoot'")
	}

	// Note: Abs error is not tested
	cases := []struct {
		val      rspec.Spec
		platform string
		expected specerror.Code
	}{
		{rspec.Spec{Windows: &rspec.Windows{HyperV: &rspec.WindowsHyperV{}}, Root: &rspec.Root{}}, "windows", specerror.RootOnHyperV},
		{rspec.Spec{Windows: &rspec.Windows{HyperV: &rspec.WindowsHyperV{}}, Root: nil}, "windows", specerror.NonError},
		{rspec.Spec{Root: nil}, "linux", specerror.RootOnNonHyperV},
		{rspec.Spec{Root: &rspec.Root{Path: "maverick-rootfs"}}, "linux", specerror.PathName},
		{rspec.Spec{Root: &rspec.Root{Path: "rootfs"}}, "linux", specerror.NonError},
		{rspec.Spec{Root: &rspec.Root{Path: filepath.Join(tmpBundle, rootfsNonExists)}}, "linux", specerror.PathExistence},
		{rspec.Spec{Root: &rspec.Root{Path: filepath.Join(tmpBundle, rootfsNonDir)}}, "linux", specerror.PathExistence},
		{rspec.Spec{Root: &rspec.Root{Path: filepath.Join(tmpBundle, "rootfs")}}, "linux", specerror.NonError},
		{rspec.Spec{Root: &rspec.Root{Path: "rootfs/rootfs"}}, "linux", specerror.ArtifactsInSingleDir},
		{rspec.Spec{Root: &rspec.Root{Readonly: true}}, "windows", specerror.ReadonlyOnWindows},
	}
	for _, c := range cases {
		v := NewValidator(&c.val, tmpBundle, false, c.platform)
		err := v.CheckRoot()
		assert.Equal(t, c.expected, specerror.FindError(err, c.expected), fmt.Sprintf("Fail to check Root: %v %d", err, c.expected))
	}
}

func TestCheckSemVer(t *testing.T) {
	cases := []struct {
		val      string
		expected specerror.Code
	}{
		{rspec.Version, specerror.NonError},
		//FIXME: validate currently only handles rpsec.Version
		{"0.0.1", specerror.NonRFCError},
		{"invalid", specerror.SpecVersion},
	}

	for _, c := range cases {
		v := NewValidator(&rspec.Spec{Version: c.val}, "", false, "linux")
		err := v.CheckSemVer()
		assert.Equal(t, c.expected, specerror.FindError(err, c.expected), "Fail to check SemVer "+c.val)
	}
}
<|endoftext|>


<commit_before>package raft

import (
	"fmt"
	"github.com/arpith/mmapd/db"
)

type returnChanMessage struct {
	err  error
	json string
}

type readRequest struct {
	key        string
	returnChan chan db.ReturnChanMessage
}

type writeRequest struct {
	key        string
	value      string
	returnChan chan db.ReturnChanMessage
}

func (s *server) handleReadRequest(req readRequest) {
	key := req.key
	c := req.returnChan
	m := db.ReadChanMessage{key, c}
	s.db.ReadChan <- m
}

func (s *server) handleWriteRequest(req writeRequest) {
	key := req.key
	value := req.value
	command := "SET " + key + " " + value
	c := make(chan bool)
	go s.appendEntry(command, c)
	<-c
	m := db.WriteChanMessage{key, value, req.returnChan}
	s.db.WriteChan <- m
}
<commit_msg>remove fmt dependency<commit_after>package raft

import (
	"github.com/arpith/mmapd/db"
)

type returnChanMessage struct {
	err  error
	json string
}

type readRequest struct {
	key        string
	returnChan chan db.ReturnChanMessage
}

type writeRequest struct {
	key        string
	value      string
	returnChan chan db.ReturnChanMessage
}

func (s *server) handleReadRequest(req readRequest) {
	key := req.key
	c := req.returnChan
	m := db.ReadChanMessage{key, c}
	s.db.ReadChan <- m
}

func (s *server) handleWriteRequest(req writeRequest) {
	key := req.key
	value := req.value
	command := "SET " + key + " " + value
	c := make(chan bool)
	go s.appendEntry(command, c)
	<-c
	m := db.WriteChanMessage{key, value, req.returnChan}
	s.db.WriteChan <- m
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// An integration test that uses a real S3 account. Run as follows:
//
//     go run integration_test/*.go \
//         -key_id <key ID> \
//         -bucket <bucket> \
//         -region s3-ap-northeast-1.amazonaws.com
//
// Before doing this, create an empty bucket (or delete the contents of an
// existing bucket) using the S3 management console.

package main

import (
	"github.com/jacobsa/aws/s3"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type BucketTest struct {
	bucket s3.Bucket
}

func init() { RegisterTestSuite(&BucketTest{}) }

func (t *BucketTest) ensureDeleted(key string) {
	err := t.bucket.DeleteObject(key)
	AssertEq(nil, err, "Couldn't delete object: %s", key)
}

func (t *BucketTest) SetUp(i *TestInfo) {
	var err error

	// Open a bucket.
	t.bucket, err = s3.OpenBucket(*g_bucketName, s3.Region(*g_region), g_accessKey)
	AssertEq(nil, err)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *BucketTest) WrongAccessKeySecret() {
	// Open a bucket with the wrong key.
	wrongKey := g_accessKey
	wrongKey.Secret += "taco"

	bucket, err := s3.OpenBucket(*g_bucketName, s3.Region(*g_region), wrongKey)
	AssertEq(nil, err)

	// Attempt to do something.
	_, err = bucket.ListKeys("")
	ExpectThat(err, Error(HasSubstr("signature")))
}

func (t *BucketTest) InvalidUtf8Keys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) LongKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) NullBytesInKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) NonGraphicalCharacterInKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) EmptyKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) GetNonExistentObject() {
	_, err := t.bucket.GetObject("some_key")

	ExpectThat(err, Error(HasSubstr("404")))
	ExpectThat(err, Error(HasSubstr("some_key")))
	ExpectThat(err, Error(HasSubstr("exist")))
}

func (t *BucketTest) StoreThenGetEmptyObject() {
	key := "some_key"
	defer t.ensureDeleted(key)

	data := []byte{}

	// Store
	err := t.bucket.StoreObject(key, data)
	AssertEq(nil, err)

	// Get
	returnedData, err := t.bucket.GetObject(key)
	AssertEq(nil, err)
	ExpectThat(returnedData, DeepEquals(data))
}

func (t *BucketTest) StoreThenGetNonEmptyObject() {
	key := "some_key"
	defer t.ensureDeleted(key)

	data := []byte{0x17, 0x19, 0x00, 0x02, 0x03}

	// Store
	err := t.bucket.StoreObject(key, data)
	AssertEq(nil, err)

	// Get
	returnedData, err := t.bucket.GetObject(key)
	AssertEq(nil, err)
	ExpectThat(returnedData, DeepEquals(data))
}

func (t *BucketTest) ListEmptyBucket() {
	var keys []string
	var err error

	// From start.
	keys, err = t.bucket.ListKeys("")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())

	// From middle.
	keys, err = t.bucket.ListKeys("foo")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())
}

func (t *BucketTest) ListWithInvalidUtf8Minimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListWithLongMinimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListWithNullByteInMinimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListFewKeys() {
	var keys []string
	var err error

	// Create several keys. S3 returns keys in an XML 1.0 document, and according
	// to Section 2.2 of the spec the smallest legal character is #x9, so a
	// string's successor in that space of strings is computed by appending \x09.
	//
	// S3 will actually allow you to create a smaller key, e.g. "bar\x01", but
	// Go's xml package will then refuse to parse its LIST responses.
	toCreate := []string{
		"foo",
		"bar",
		"bar\x09",
		"bar\x09\x09",
		"baz",
	}

	for _, key := range toCreate {
		defer t.ensureDeleted(key)
		err := t.bucket.StoreObject(key, []byte{})
		AssertEq(nil, err, "Creating object: %s", key)
	}

	// From start.
	keys, err = t.bucket.ListKeys("")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"bar",
		"bar\x09",
		"bar\x09\x09",
		"baz",
		"foo",
	))

	// Just before bar\x09.
	keys, err = t.bucket.ListKeys("bar")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"bar\x09",
		"bar\x09\x09",
		"baz",
		"foo",
	))

	// At bar\x09.
	keys, err = t.bucket.ListKeys("bar\x09")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"bar\x09\x09",
		"baz",
		"foo",
	))

	// Just after bar\x09.
	keys, err = t.bucket.ListKeys("bar\x09\x09")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"baz",
		"foo",
	))

	// At last key.
	keys, err = t.bucket.ListKeys("foo")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())

	// Just after last key.
	keys, err = t.bucket.ListKeys("foo\x09")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())

	// Well after last key.
	keys, err = t.bucket.ListKeys("qux")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())
}

func (t *BucketTest) ListManyKeys() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) KeysWithSpecialCharacters() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) DeleteNonExistentObject() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) DeleteThenListAndGetObject() {
	ExpectFalse(true, "TODO")
}
<commit_msg>Added some parallelism to key creation.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// An integration test that uses a real S3 account. Run as follows:
//
//     go run integration_test/*.go \
//         -key_id <key ID> \
//         -bucket <bucket> \
//         -region s3-ap-northeast-1.amazonaws.com
//
// Before doing this, create an empty bucket (or delete the contents of an
// existing bucket) using the S3 management console.

package main

import (
	"github.com/jacobsa/aws/s3"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Run the supplied function for every integer in [0, n) with some degree of
// parallelism, returning an error if any invocation returns an error.
func runForRange(n int, f func (int) error) (err error) {
		// Set up channels. The work channel should be buffered so that we don't
		// have to block writing to it before checking for errors below. The error
		// channel must be buffered so that no worker goroutine gets stuck writing
		// a result to it and never returns. The stop channel must not be buffered
		// so that we can be sure that no more work will be processed when we
		// return below.
		work := make(chan int, n)
		errs := make(chan error, n)
		stop := make(chan bool)

		// Launch worker functions that attempt to do work, returning if a read
		// from the stop channel succeeds.
		processWork := func() {
			for {
				select {
				case i := <-work:
					errs<- f(i)
				case <-stop:
					return
				}
			}
		}

		const numWorkers = 17
		for i := 0; i < numWorkers; i++ {
			go processWork()
		}

		// Feed the workers work.
		for i := 0; i < n; i++ {
			work<- i
		}

		// Read results, stopping immediately if there is an error.
		for i := 0; i < n; i++ {
			err = <-errs
			if err != nil {
				break
			}
		}

		// Stop all of the workers, and wait for them to stop. This ensures that
		// no piece of work is in progress when we return.
		for i := 0; i < numWorkers; i++ {
			stop<- true
		}

		return
	}

type BucketTest struct {
	bucket s3.Bucket
}

func init() { RegisterTestSuite(&BucketTest{}) }

func (t *BucketTest) ensureDeleted(key string) {
	err := t.bucket.DeleteObject(key)
	AssertEq(nil, err, "Couldn't delete object: %s", key)
}

func (t *BucketTest) SetUp(i *TestInfo) {
	var err error

	// Open a bucket.
	t.bucket, err = s3.OpenBucket(*g_bucketName, s3.Region(*g_region), g_accessKey)
	AssertEq(nil, err)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *BucketTest) WrongAccessKeySecret() {
	// Open a bucket with the wrong key.
	wrongKey := g_accessKey
	wrongKey.Secret += "taco"

	bucket, err := s3.OpenBucket(*g_bucketName, s3.Region(*g_region), wrongKey)
	AssertEq(nil, err)

	// Attempt to do something.
	_, err = bucket.ListKeys("")
	ExpectThat(err, Error(HasSubstr("signature")))
}

func (t *BucketTest) InvalidUtf8Keys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) LongKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) NullBytesInKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) NonGraphicalCharacterInKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) EmptyKeys() {
	ExpectEq("TODO", "")
}

func (t *BucketTest) GetNonExistentObject() {
	_, err := t.bucket.GetObject("some_key")

	ExpectThat(err, Error(HasSubstr("404")))
	ExpectThat(err, Error(HasSubstr("some_key")))
	ExpectThat(err, Error(HasSubstr("exist")))
}

func (t *BucketTest) StoreThenGetEmptyObject() {
	key := "some_key"
	defer t.ensureDeleted(key)

	data := []byte{}

	// Store
	err := t.bucket.StoreObject(key, data)
	AssertEq(nil, err)

	// Get
	returnedData, err := t.bucket.GetObject(key)
	AssertEq(nil, err)
	ExpectThat(returnedData, DeepEquals(data))
}

func (t *BucketTest) StoreThenGetNonEmptyObject() {
	key := "some_key"
	defer t.ensureDeleted(key)

	data := []byte{0x17, 0x19, 0x00, 0x02, 0x03}

	// Store
	err := t.bucket.StoreObject(key, data)
	AssertEq(nil, err)

	// Get
	returnedData, err := t.bucket.GetObject(key)
	AssertEq(nil, err)
	ExpectThat(returnedData, DeepEquals(data))
}

func (t *BucketTest) ListEmptyBucket() {
	var keys []string
	var err error

	// From start.
	keys, err = t.bucket.ListKeys("")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())

	// From middle.
	keys, err = t.bucket.ListKeys("foo")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())
}

func (t *BucketTest) ListWithInvalidUtf8Minimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListWithLongMinimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListWithNullByteInMinimum() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) ListFewKeys() {
	var keys []string
	var err error

	// Create several keys. S3 returns keys in an XML 1.0 document, and according
	// to Section 2.2 of the spec the smallest legal character is #x9, so a
	// string's successor in that space of strings is computed by appending \x09.
	//
	// S3 will actually allow you to create a smaller key, e.g. "bar\x01", but
	// Go's xml package will then refuse to parse its LIST responses.
	toCreate := []string{
		"foo",
		"bar",
		"bar\x09",
		"bar\x09\x09",
		"baz",
	}

	defer runForRange(len(toCreate), func(i int) error {
		key := toCreate[i]
		t.ensureDeleted(key)
		return nil
	})

	err = runForRange(len(toCreate), func(i int) error {
		key := toCreate[i]
		return t.bucket.StoreObject(key, []byte{})
	})

	AssertEq(nil, err)

	// From start.
	keys, err = t.bucket.ListKeys("")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"bar",
		"bar\x09",
		"bar\x09\x09",
		"baz",
		"foo",
	))

	// Just before bar\x09.
	keys, err = t.bucket.ListKeys("bar")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"bar\x09",
		"bar\x09\x09",
		"baz",
		"foo",
	))

	// At bar\x09.
	keys, err = t.bucket.ListKeys("bar\x09")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"bar\x09\x09",
		"baz",
		"foo",
	))

	// Just after bar\x09.
	keys, err = t.bucket.ListKeys("bar\x09\x09")
	AssertEq(nil, err)
	ExpectThat(
		keys,
		ElementsAre(
		"baz",
		"foo",
	))

	// At last key.
	keys, err = t.bucket.ListKeys("foo")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())

	// Just after last key.
	keys, err = t.bucket.ListKeys("foo\x09")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())

	// Well after last key.
	keys, err = t.bucket.ListKeys("qux")
	AssertEq(nil, err)
	ExpectThat(keys, ElementsAre())
}

func (t *BucketTest) ListManyKeys() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) KeysWithSpecialCharacters() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) DeleteNonExistentObject() {
	ExpectFalse(true, "TODO")
}

func (t *BucketTest) DeleteThenListAndGetObject() {
	ExpectFalse(true, "TODO")
}
<|endoftext|>


<commit_before>// The digitalocean package contains a packer.Builder implementation
// that builds DigitalOcean images (snapshots).

package digitalocean

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/mitchellh/mapstructure"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/builder/common"
	"github.com/mitchellh/packer/packer"
	"log"
	"strconv"
	"text/template"
	"time"
)

// The unique id for the builder
const BuilderId = "pearkes.digitalocean"

type snapshotNameData struct {
	CreateTime string
}

// Configuration tells the builder the credentials
// to use while communicating with DO and describes the image
// you are creating
type config struct {
	ClientID string `mapstructure:"client_id"`
	APIKey   string `mapstructure:"api_key"`
	RegionID uint   `mapstructure:"region_id"`
	SizeID   uint   `mapstructure:"size_id"`
	ImageID  uint   `mapstructure:"image_id"`

	SnapshotName string
	SSHUsername  string `mapstructure:"ssh_username"`
	SSHPort      uint   `mapstructure:"ssh_port"`
	SSHTimeout   time.Duration
	EventDelay   time.Duration

	PackerDebug bool `mapstructure:"packer_debug"`

	RawSnapshotName string `mapstructure:"snapshot_name"`
	RawSSHTimeout   string `mapstructure:"ssh_timeout"`
	RawEventDelay   string `mapstructure:"event_delay"`
}

type Builder struct {
	config config
	runner multistep.Runner
}

func (b *Builder) Prepare(raws ...interface{}) error {
	for _, raw := range raws {
		err := mapstructure.Decode(raw, &b.config)
		if err != nil {
			return err
		}
	}

	// Optional configuration with defaults
	//
	if b.config.RegionID == 0 {
		// Default to Region "New York"
		b.config.RegionID = 1
	}

	if b.config.SizeID == 0 {
		// Default to 512mb, the smallest droplet size
		b.config.SizeID = 66
	}

	if b.config.ImageID == 0 {
		// Default to base image "Ubuntu 12.04 x64 Server (id: 284203)"
		b.config.ImageID = 284203
	}

	if b.config.SSHUsername == "" {
		// Default to "root". You can override this if your
		// SourceImage has a different user account then the DO default
		b.config.SSHUsername = "root"
	}

	if b.config.SSHPort == 0 {
		// Default to port 22 per DO default
		b.config.SSHPort = 22
	}

	if b.config.RawSnapshotName == "" {
		// Default to packer-{{ unix timestamp (utc) }}
		b.config.RawSnapshotName = "packer-{{.CreateTime}}"
	}

	if b.config.RawSSHTimeout == "" {
		// Default to 1 minute timeouts
		b.config.RawSSHTimeout = "1m"
	}

	if b.config.RawEventDelay == "" {
		// Default to 5 second delays after creating events
		// to allow DO to process
		b.config.RawEventDelay = "5s"
	}

	// A list of errors on the configuration
	errs := make([]error, 0)

	// Required configurations that will display errors if not set
	//
	if b.config.ClientID == "" {
		errs = append(errs, errors.New("a client_id must be specified"))
	}

	if b.config.APIKey == "" {
		errs = append(errs, errors.New("an api_key must be specified"))
	}

	timeout, err := time.ParseDuration(b.config.RawSSHTimeout)
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err))
	}
	b.config.SSHTimeout = timeout

	delay, err := time.ParseDuration(b.config.RawEventDelay)
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed parsing event_delay: %s", err))
	}
	b.config.EventDelay = delay

	// Parse the name of the snapshot
	snapNameBuf := new(bytes.Buffer)
	tData := snapshotNameData{
		strconv.FormatInt(time.Now().UTC().Unix(), 10),
	}
	t, err := template.New("snapshot").Parse(b.config.RawSnapshotName)
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed parsing snapshot_name: %s", err))
	}
	t.Execute(snapNameBuf, tData)
	b.config.SnapshotName = snapNameBuf.String()

	if len(errs) > 0 {
		return &packer.MultiError{errs}
	}

	log.Printf("Config: %+v", b.config)
	return nil
}

func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) {
	// Initialize the DO API client
	client := DigitalOceanClient{}.New(b.config.ClientID, b.config.APIKey)

	// Set up the state
	state := make(map[string]interface{})
	state["config"] = b.config
	state["client"] = client
	state["hook"] = hook
	state["ui"] = ui

	// Build the steps
	steps := []multistep.Step{
		new(stepCreateSSHKey),
		new(stepCreateDroplet),
		new(stepDropletInfo),
		new(stepConnectSSH),
		new(stepProvision),
		new(stepPowerOff),
		new(stepSnapshot),
	}

	// Run the steps
	if b.config.PackerDebug {
		b.runner = &multistep.DebugRunner{
			Steps:   steps,
			PauseFn: common.MultistepDebugFn(ui),
		}
	} else {
		b.runner = &multistep.BasicRunner{Steps: steps}
	}

	b.runner.Run(state)

	return &Artifact{b.config.SnapshotName}, nil
}

func (b *Builder) Cancel() {
	if b.runner != nil {
		log.Println("Cancelling the step runner...")
		b.runner.Cancel()
	}
}
<commit_msg>builder/digitalocean: only execute the snapshotname template if parsed<commit_after>// The digitalocean package contains a packer.Builder implementation
// that builds DigitalOcean images (snapshots).

package digitalocean

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/mitchellh/mapstructure"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/builder/common"
	"github.com/mitchellh/packer/packer"
	"log"
	"strconv"
	"text/template"
	"time"
)

// The unique id for the builder
const BuilderId = "pearkes.digitalocean"

type snapshotNameData struct {
	CreateTime string
}

// Configuration tells the builder the credentials
// to use while communicating with DO and describes the image
// you are creating
type config struct {
	ClientID string `mapstructure:"client_id"`
	APIKey   string `mapstructure:"api_key"`
	RegionID uint   `mapstructure:"region_id"`
	SizeID   uint   `mapstructure:"size_id"`
	ImageID  uint   `mapstructure:"image_id"`

	SnapshotName string
	SSHUsername  string `mapstructure:"ssh_username"`
	SSHPort      uint   `mapstructure:"ssh_port"`
	SSHTimeout   time.Duration
	EventDelay   time.Duration

	PackerDebug bool `mapstructure:"packer_debug"`

	RawSnapshotName string `mapstructure:"snapshot_name"`
	RawSSHTimeout   string `mapstructure:"ssh_timeout"`
	RawEventDelay   string `mapstructure:"event_delay"`
}

type Builder struct {
	config config
	runner multistep.Runner
}

func (b *Builder) Prepare(raws ...interface{}) error {
	for _, raw := range raws {
		err := mapstructure.Decode(raw, &b.config)
		if err != nil {
			return err
		}
	}

	// Optional configuration with defaults
	//
	if b.config.RegionID == 0 {
		// Default to Region "New York"
		b.config.RegionID = 1
	}

	if b.config.SizeID == 0 {
		// Default to 512mb, the smallest droplet size
		b.config.SizeID = 66
	}

	if b.config.ImageID == 0 {
		// Default to base image "Ubuntu 12.04 x64 Server (id: 284203)"
		b.config.ImageID = 284203
	}

	if b.config.SSHUsername == "" {
		// Default to "root". You can override this if your
		// SourceImage has a different user account then the DO default
		b.config.SSHUsername = "root"
	}

	if b.config.SSHPort == 0 {
		// Default to port 22 per DO default
		b.config.SSHPort = 22
	}

	if b.config.RawSnapshotName == "" {
		// Default to packer-{{ unix timestamp (utc) }}
		b.config.RawSnapshotName = "packer-{{.CreateTime}}"
	}

	if b.config.RawSSHTimeout == "" {
		// Default to 1 minute timeouts
		b.config.RawSSHTimeout = "1m"
	}

	if b.config.RawEventDelay == "" {
		// Default to 5 second delays after creating events
		// to allow DO to process
		b.config.RawEventDelay = "5s"
	}

	// A list of errors on the configuration
	errs := make([]error, 0)

	// Required configurations that will display errors if not set
	//
	if b.config.ClientID == "" {
		errs = append(errs, errors.New("a client_id must be specified"))
	}

	if b.config.APIKey == "" {
		errs = append(errs, errors.New("an api_key must be specified"))
	}

	timeout, err := time.ParseDuration(b.config.RawSSHTimeout)
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err))
	}
	b.config.SSHTimeout = timeout

	delay, err := time.ParseDuration(b.config.RawEventDelay)
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed parsing event_delay: %s", err))
	}
	b.config.EventDelay = delay

	// Parse the name of the snapshot
	snapNameBuf := new(bytes.Buffer)
	tData := snapshotNameData{
		strconv.FormatInt(time.Now().UTC().Unix(), 10),
	}
	t, err := template.New("snapshot").Parse(b.config.RawSnapshotName)
	if err != nil {
		errs = append(errs, fmt.Errorf("Failed parsing snapshot_name: %s", err))
	} else {
		t.Execute(snapNameBuf, tData)
		b.config.SnapshotName = snapNameBuf.String()
	}

	if len(errs) > 0 {
		return &packer.MultiError{errs}
	}

	log.Printf("Config: %+v", b.config)
	return nil
}

func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) {
	// Initialize the DO API client
	client := DigitalOceanClient{}.New(b.config.ClientID, b.config.APIKey)

	// Set up the state
	state := make(map[string]interface{})
	state["config"] = b.config
	state["client"] = client
	state["hook"] = hook
	state["ui"] = ui

	// Build the steps
	steps := []multistep.Step{
		new(stepCreateSSHKey),
		new(stepCreateDroplet),
		new(stepDropletInfo),
		new(stepConnectSSH),
		new(stepProvision),
		new(stepPowerOff),
		new(stepSnapshot),
	}

	// Run the steps
	if b.config.PackerDebug {
		b.runner = &multistep.DebugRunner{
			Steps:   steps,
			PauseFn: common.MultistepDebugFn(ui),
		}
	} else {
		b.runner = &multistep.BasicRunner{Steps: steps}
	}

	b.runner.Run(state)

	return &Artifact{b.config.SnapshotName}, nil
}

func (b *Builder) Cancel() {
	if b.runner != nil {
		log.Println("Cancelling the step runner...")
		b.runner.Cancel()
	}
}
<|endoftext|>