<commit_before>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package channelconfig

import (
	"fmt"
	"io/ioutil"
	"math"

	"github.com/golang/protobuf/proto"
	cb "github.com/hyperledger/fabric-protos-go/common"
	mspprotos "github.com/hyperledger/fabric-protos-go/msp"
	ab "github.com/hyperledger/fabric-protos-go/orderer"
	"github.com/hyperledger/fabric-protos-go/orderer/etcdraft"
	pb "github.com/hyperledger/fabric-protos-go/peer"
	"github.com/hyperledger/fabric/bccsp"
	"github.com/hyperledger/fabric/bccsp/factory"
	"github.com/hyperledger/fabric/protoutil"
	"github.com/pkg/errors"
)

const (
	// ReadersPolicyKey is the key used for the read policy
	ReadersPolicyKey = "Readers"

	// WritersPolicyKey is the key used for the read policy
	WritersPolicyKey = "Writers"

	// AdminsPolicyKey is the key used for the read policy
	AdminsPolicyKey = "Admins"

	defaultHashingAlgorithm = bccsp.SHA256

	defaultBlockDataHashingStructureWidth = math.MaxUint32
)

// ConfigValue defines a common representation for different *cb.ConfigValue values.
type ConfigValue interface {
	// Key is the key this value should be stored in the *cb.ConfigGroup.Values map.
	Key() string

	// Value is the message which should be marshaled to opaque bytes for the *cb.ConfigValue.value.
	Value() proto.Message
}

// StandardConfigValue implements the ConfigValue interface.
type StandardConfigValue struct {
	key   string
	value proto.Message
}

// Key is the key this value should be stored in the *cb.ConfigGroup.Values map.
func (scv *StandardConfigValue) Key() string {
	return scv.key
}

// Value is the message which should be marshaled to opaque bytes for the *cb.ConfigValue.value.
func (scv *StandardConfigValue) Value() proto.Message {
	return scv.value
}

// ConsortiumValue returns the config definition for the consortium name.
// It is a value for the channel group.
func ConsortiumValue(name string) *StandardConfigValue {
	return &StandardConfigValue{
		key: ConsortiumKey,
		value: &cb.Consortium{
			Name: name,
		},
	}
}

// HashingAlgorithm returns the only currently valid hashing algorithm.
// It is a value for the /Channel group.
func HashingAlgorithmValue() *StandardConfigValue {
	return &StandardConfigValue{
		key: HashingAlgorithmKey,
		value: &cb.HashingAlgorithm{
			Name: defaultHashingAlgorithm,
		},
	}
}

// BlockDataHashingStructureValue returns the only currently valid block data hashing structure.
// It is a value for the /Channel group.
func BlockDataHashingStructureValue() *StandardConfigValue {
	return &StandardConfigValue{
		key: BlockDataHashingStructureKey,
		value: &cb.BlockDataHashingStructure{
			Width: defaultBlockDataHashingStructureWidth,
		},
	}
}

// OrdererAddressesValue returns the a config definition for the orderer addresses.
// It is a value for the /Channel group.
func OrdererAddressesValue(addresses []string) *StandardConfigValue {
	return &StandardConfigValue{
		key: OrdererAddressesKey,
		value: &cb.OrdererAddresses{
			Addresses: addresses,
		},
	}
}

// ConsensusTypeValue returns the config definition for the orderer consensus type.
// It is a value for the /Channel/Orderer group.
func ConsensusTypeValue(consensusType string, consensusMetadata []byte) *StandardConfigValue {
	return &StandardConfigValue{
		key: ConsensusTypeKey,
		value: &ab.ConsensusType{
			Type:     consensusType,
			Metadata: consensusMetadata,
		},
	}
}

// BatchSizeValue returns the config definition for the orderer batch size.
// It is a value for the /Channel/Orderer group.
func BatchSizeValue(maxMessages, absoluteMaxBytes, preferredMaxBytes uint32) *StandardConfigValue {
	return &StandardConfigValue{
		key: BatchSizeKey,
		value: &ab.BatchSize{
			MaxMessageCount:   maxMessages,
			AbsoluteMaxBytes:  absoluteMaxBytes,
			PreferredMaxBytes: preferredMaxBytes,
		},
	}
}

// BatchTimeoutValue returns the config definition for the orderer batch timeout.
// It is a value for the /Channel/Orderer group.
func BatchTimeoutValue(timeout string) *StandardConfigValue {
	return &StandardConfigValue{
		key: BatchTimeoutKey,
		value: &ab.BatchTimeout{
			Timeout: timeout,
		},
	}
}

// ChannelRestrictionsValue returns the config definition for the orderer channel restrictions.
// It is a value for the /Channel/Orderer group.
func ChannelRestrictionsValue(maxChannelCount uint64) *StandardConfigValue {
	return &StandardConfigValue{
		key: ChannelRestrictionsKey,
		value: &ab.ChannelRestrictions{
			MaxCount: maxChannelCount,
		},
	}
}

// KafkaBrokersValue returns the config definition for the addresses of the ordering service's Kafka brokers.
// It is a value for the /Channel/Orderer group.
func KafkaBrokersValue(brokers []string) *StandardConfigValue {
	return &StandardConfigValue{
		key: KafkaBrokersKey,
		value: &ab.KafkaBrokers{
			Brokers: brokers,
		},
	}
}

// MSPValue returns the config definition for an MSP.
// It is a value for the /Channel/Orderer/*, /Channel/Application/*, and /Channel/Consortiums/*/*/* groups.
func MSPValue(mspDef *mspprotos.MSPConfig) *StandardConfigValue {
	return &StandardConfigValue{
		key:   MSPKey,
		value: mspDef,
	}
}

// CapabilitiesValue returns the config definition for a a set of capabilities.
// It is a value for the /Channel/Orderer, Channel/Application/, and /Channel groups.
func CapabilitiesValue(capabilities map[string]bool) *StandardConfigValue {
	c := &cb.Capabilities{
		Capabilities: make(map[string]*cb.Capability),
	}

	for capability, required := range capabilities {
		if !required {
			continue
		}
		c.Capabilities[capability] = &cb.Capability{}
	}

	return &StandardConfigValue{
		key:   CapabilitiesKey,
		value: c,
	}
}

// EndpointsValue returns the config definition for the orderer addresses at an org scoped level.
// It is a value for the /Channel/Orderer/<OrgName> group.
func EndpointsValue(addresses []string) *StandardConfigValue {
	return &StandardConfigValue{
		key: EndpointsKey,
		value: &cb.OrdererAddresses{
			Addresses: addresses,
		},
	}
}

// AnchorPeersValue returns the config definition for an org's anchor peers.
// It is a value for the /Channel/Application/*.
func AnchorPeersValue(anchorPeers []*pb.AnchorPeer) *StandardConfigValue {
	return &StandardConfigValue{
		key:   AnchorPeersKey,
		value: &pb.AnchorPeers{AnchorPeers: anchorPeers},
	}
}

// ChannelCreationPolicyValue returns the config definition for a consortium's channel creation policy
// It is a value for the /Channel/Consortiums/*/*.
func ChannelCreationPolicyValue(policy *cb.Policy) *StandardConfigValue {
	return &StandardConfigValue{
		key:   ChannelCreationPolicyKey,
		value: policy,
	}
}

// ACLValues returns the config definition for an applications resources based ACL definitions.
// It is a value for the /Channel/Application/.
func ACLValues(acls map[string]string) *StandardConfigValue {
	a := &pb.ACLs{
		Acls: make(map[string]*pb.APIResource),
	}

	for apiResource, policyRef := range acls {
		a.Acls[apiResource] = &pb.APIResource{PolicyRef: policyRef}
	}

	return &StandardConfigValue{
		key:   ACLsKey,
		value: a,
	}
}

// ValidateCapabilities validates whether the peer can meet the capabilities requirement in the given config block
func ValidateCapabilities(block *cb.Block, bccsp bccsp.BCCSP) error {
	cc, err := extractChannelConfig(block, bccsp)
	if err != nil {
		return err
	}
	// Check the channel top-level capabilities
	if err := cc.Capabilities().Supported(); err != nil {
		return err
	}

	// Check the application capabilities
	return cc.ApplicationConfig().Capabilities().Supported()
}

// ExtractMSPIDsForApplicationOrgs extracts MSPIDs for application organizations
func ExtractMSPIDsForApplicationOrgs(block *cb.Block, bccsp bccsp.BCCSP) ([]string, error) {
	cc, err := extractChannelConfig(block, factory.GetDefault())
	if err != nil {
		return nil, err
	}

	if cc.ApplicationConfig() == nil {
		return nil, errors.Errorf("could not get application config for the channel")
	}
	orgs := cc.ApplicationConfig().Organizations()
	mspids := make([]string, 0, len(orgs))
	for _, org := range orgs {
		mspids = append(mspids, org.MSPID())
	}
	return mspids, nil
}

func extractChannelConfig(block *cb.Block, bccsp bccsp.BCCSP) (*ChannelConfig, error) {
	envelopeConfig, err := protoutil.ExtractEnvelope(block, 0)
	if err != nil {
		return nil, errors.WithMessage(err, "malformed configuration block")
	}

	configEnv := &cb.ConfigEnvelope{}
	_, err = protoutil.UnmarshalEnvelopeOfType(envelopeConfig, cb.HeaderType_CONFIG, configEnv)
	if err != nil {
		return nil, errors.WithMessage(err, "malformed configuration envelope")
	}

	if configEnv.Config == nil {
		return nil, errors.New("no config found in envelope")
	}

	if configEnv.Config.ChannelGroup == nil {
		return nil, errors.New("no channel configuration found in the config block")
	}

	if configEnv.Config.ChannelGroup.Groups == nil {
		return nil, errors.New("no channel configuration groups are available")
	}

	_, exists := configEnv.Config.ChannelGroup.Groups[ApplicationGroupKey]
	if !exists {
		return nil, errors.Errorf("invalid configuration block, missing %s configuration group", ApplicationGroupKey)
	}

	cc, err := NewChannelConfig(configEnv.Config.ChannelGroup, bccsp)
	if err != nil {
		return nil, errors.WithMessage(err, "no valid channel configuration found")
	}
	return cc, nil
}

// MarshalEtcdRaftMetadata serializes etcd RAFT metadata.
func MarshalEtcdRaftMetadata(md *etcdraft.ConfigMetadata) ([]byte, error) {
	copyMd := proto.Clone(md).(*etcdraft.ConfigMetadata)
	for _, c := range copyMd.Consenters {
		// Expect the user to set the config value for client/server certs to the
		// path where they are persisted locally, then load these files to memory.
		clientCert, err := ioutil.ReadFile(string(c.GetClientTlsCert()))
		if err != nil {
			return nil, fmt.Errorf("cannot load client cert for consenter %s:%d: %s", c.GetHost(), c.GetPort(), err)
		}
		c.ClientTlsCert = clientCert

		serverCert, err := ioutil.ReadFile(string(c.GetServerTlsCert()))
		if err != nil {
			return nil, fmt.Errorf("cannot load server cert for consenter %s:%d: %s", c.GetHost(), c.GetPort(), err)
		}
		c.ServerTlsCert = serverCert
	}
	return proto.Marshal(copyMd)
}
<commit_msg>Wire function parameter to its actual usage (#2499)<commit_after>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package channelconfig

import (
	"fmt"
	"io/ioutil"
	"math"

	"github.com/golang/protobuf/proto"
	cb "github.com/hyperledger/fabric-protos-go/common"
	mspprotos "github.com/hyperledger/fabric-protos-go/msp"
	ab "github.com/hyperledger/fabric-protos-go/orderer"
	"github.com/hyperledger/fabric-protos-go/orderer/etcdraft"
	pb "github.com/hyperledger/fabric-protos-go/peer"
	"github.com/hyperledger/fabric/bccsp"
	"github.com/hyperledger/fabric/protoutil"
	"github.com/pkg/errors"
)

const (
	// ReadersPolicyKey is the key used for the read policy
	ReadersPolicyKey = "Readers"

	// WritersPolicyKey is the key used for the read policy
	WritersPolicyKey = "Writers"

	// AdminsPolicyKey is the key used for the read policy
	AdminsPolicyKey = "Admins"

	defaultHashingAlgorithm = bccsp.SHA256

	defaultBlockDataHashingStructureWidth = math.MaxUint32
)

// ConfigValue defines a common representation for different *cb.ConfigValue values.
type ConfigValue interface {
	// Key is the key this value should be stored in the *cb.ConfigGroup.Values map.
	Key() string

	// Value is the message which should be marshaled to opaque bytes for the *cb.ConfigValue.value.
	Value() proto.Message
}

// StandardConfigValue implements the ConfigValue interface.
type StandardConfigValue struct {
	key   string
	value proto.Message
}

// Key is the key this value should be stored in the *cb.ConfigGroup.Values map.
func (scv *StandardConfigValue) Key() string {
	return scv.key
}

// Value is the message which should be marshaled to opaque bytes for the *cb.ConfigValue.value.
func (scv *StandardConfigValue) Value() proto.Message {
	return scv.value
}

// ConsortiumValue returns the config definition for the consortium name.
// It is a value for the channel group.
func ConsortiumValue(name string) *StandardConfigValue {
	return &StandardConfigValue{
		key: ConsortiumKey,
		value: &cb.Consortium{
			Name: name,
		},
	}
}

// HashingAlgorithm returns the only currently valid hashing algorithm.
// It is a value for the /Channel group.
func HashingAlgorithmValue() *StandardConfigValue {
	return &StandardConfigValue{
		key: HashingAlgorithmKey,
		value: &cb.HashingAlgorithm{
			Name: defaultHashingAlgorithm,
		},
	}
}

// BlockDataHashingStructureValue returns the only currently valid block data hashing structure.
// It is a value for the /Channel group.
func BlockDataHashingStructureValue() *StandardConfigValue {
	return &StandardConfigValue{
		key: BlockDataHashingStructureKey,
		value: &cb.BlockDataHashingStructure{
			Width: defaultBlockDataHashingStructureWidth,
		},
	}
}

// OrdererAddressesValue returns the a config definition for the orderer addresses.
// It is a value for the /Channel group.
func OrdererAddressesValue(addresses []string) *StandardConfigValue {
	return &StandardConfigValue{
		key: OrdererAddressesKey,
		value: &cb.OrdererAddresses{
			Addresses: addresses,
		},
	}
}

// ConsensusTypeValue returns the config definition for the orderer consensus type.
// It is a value for the /Channel/Orderer group.
func ConsensusTypeValue(consensusType string, consensusMetadata []byte) *StandardConfigValue {
	return &StandardConfigValue{
		key: ConsensusTypeKey,
		value: &ab.ConsensusType{
			Type:     consensusType,
			Metadata: consensusMetadata,
		},
	}
}

// BatchSizeValue returns the config definition for the orderer batch size.
// It is a value for the /Channel/Orderer group.
func BatchSizeValue(maxMessages, absoluteMaxBytes, preferredMaxBytes uint32) *StandardConfigValue {
	return &StandardConfigValue{
		key: BatchSizeKey,
		value: &ab.BatchSize{
			MaxMessageCount:   maxMessages,
			AbsoluteMaxBytes:  absoluteMaxBytes,
			PreferredMaxBytes: preferredMaxBytes,
		},
	}
}

// BatchTimeoutValue returns the config definition for the orderer batch timeout.
// It is a value for the /Channel/Orderer group.
func BatchTimeoutValue(timeout string) *StandardConfigValue {
	return &StandardConfigValue{
		key: BatchTimeoutKey,
		value: &ab.BatchTimeout{
			Timeout: timeout,
		},
	}
}

// ChannelRestrictionsValue returns the config definition for the orderer channel restrictions.
// It is a value for the /Channel/Orderer group.
func ChannelRestrictionsValue(maxChannelCount uint64) *StandardConfigValue {
	return &StandardConfigValue{
		key: ChannelRestrictionsKey,
		value: &ab.ChannelRestrictions{
			MaxCount: maxChannelCount,
		},
	}
}

// KafkaBrokersValue returns the config definition for the addresses of the ordering service's Kafka brokers.
// It is a value for the /Channel/Orderer group.
func KafkaBrokersValue(brokers []string) *StandardConfigValue {
	return &StandardConfigValue{
		key: KafkaBrokersKey,
		value: &ab.KafkaBrokers{
			Brokers: brokers,
		},
	}
}

// MSPValue returns the config definition for an MSP.
// It is a value for the /Channel/Orderer/*, /Channel/Application/*, and /Channel/Consortiums/*/*/* groups.
func MSPValue(mspDef *mspprotos.MSPConfig) *StandardConfigValue {
	return &StandardConfigValue{
		key:   MSPKey,
		value: mspDef,
	}
}

// CapabilitiesValue returns the config definition for a a set of capabilities.
// It is a value for the /Channel/Orderer, Channel/Application/, and /Channel groups.
func CapabilitiesValue(capabilities map[string]bool) *StandardConfigValue {
	c := &cb.Capabilities{
		Capabilities: make(map[string]*cb.Capability),
	}

	for capability, required := range capabilities {
		if !required {
			continue
		}
		c.Capabilities[capability] = &cb.Capability{}
	}

	return &StandardConfigValue{
		key:   CapabilitiesKey,
		value: c,
	}
}

// EndpointsValue returns the config definition for the orderer addresses at an org scoped level.
// It is a value for the /Channel/Orderer/<OrgName> group.
func EndpointsValue(addresses []string) *StandardConfigValue {
	return &StandardConfigValue{
		key: EndpointsKey,
		value: &cb.OrdererAddresses{
			Addresses: addresses,
		},
	}
}

// AnchorPeersValue returns the config definition for an org's anchor peers.
// It is a value for the /Channel/Application/*.
func AnchorPeersValue(anchorPeers []*pb.AnchorPeer) *StandardConfigValue {
	return &StandardConfigValue{
		key:   AnchorPeersKey,
		value: &pb.AnchorPeers{AnchorPeers: anchorPeers},
	}
}

// ChannelCreationPolicyValue returns the config definition for a consortium's channel creation policy
// It is a value for the /Channel/Consortiums/*/*.
func ChannelCreationPolicyValue(policy *cb.Policy) *StandardConfigValue {
	return &StandardConfigValue{
		key:   ChannelCreationPolicyKey,
		value: policy,
	}
}

// ACLValues returns the config definition for an applications resources based ACL definitions.
// It is a value for the /Channel/Application/.
func ACLValues(acls map[string]string) *StandardConfigValue {
	a := &pb.ACLs{
		Acls: make(map[string]*pb.APIResource),
	}

	for apiResource, policyRef := range acls {
		a.Acls[apiResource] = &pb.APIResource{PolicyRef: policyRef}
	}

	return &StandardConfigValue{
		key:   ACLsKey,
		value: a,
	}
}

// ValidateCapabilities validates whether the peer can meet the capabilities requirement in the given config block
func ValidateCapabilities(block *cb.Block, bccsp bccsp.BCCSP) error {
	cc, err := extractChannelConfig(block, bccsp)
	if err != nil {
		return err
	}
	// Check the channel top-level capabilities
	if err := cc.Capabilities().Supported(); err != nil {
		return err
	}

	// Check the application capabilities
	return cc.ApplicationConfig().Capabilities().Supported()
}

// ExtractMSPIDsForApplicationOrgs extracts MSPIDs for application organizations
func ExtractMSPIDsForApplicationOrgs(block *cb.Block, bccsp bccsp.BCCSP) ([]string, error) {
	cc, err := extractChannelConfig(block, bccsp)
	if err != nil {
		return nil, err
	}

	if cc.ApplicationConfig() == nil {
		return nil, errors.Errorf("could not get application config for the channel")
	}
	orgs := cc.ApplicationConfig().Organizations()
	mspids := make([]string, 0, len(orgs))
	for _, org := range orgs {
		mspids = append(mspids, org.MSPID())
	}
	return mspids, nil
}

func extractChannelConfig(block *cb.Block, bccsp bccsp.BCCSP) (*ChannelConfig, error) {
	envelopeConfig, err := protoutil.ExtractEnvelope(block, 0)
	if err != nil {
		return nil, errors.WithMessage(err, "malformed configuration block")
	}

	configEnv := &cb.ConfigEnvelope{}
	_, err = protoutil.UnmarshalEnvelopeOfType(envelopeConfig, cb.HeaderType_CONFIG, configEnv)
	if err != nil {
		return nil, errors.WithMessage(err, "malformed configuration envelope")
	}

	if configEnv.Config == nil {
		return nil, errors.New("no config found in envelope")
	}

	if configEnv.Config.ChannelGroup == nil {
		return nil, errors.New("no channel configuration found in the config block")
	}

	if configEnv.Config.ChannelGroup.Groups == nil {
		return nil, errors.New("no channel configuration groups are available")
	}

	_, exists := configEnv.Config.ChannelGroup.Groups[ApplicationGroupKey]
	if !exists {
		return nil, errors.Errorf("invalid configuration block, missing %s configuration group", ApplicationGroupKey)
	}

	cc, err := NewChannelConfig(configEnv.Config.ChannelGroup, bccsp)
	if err != nil {
		return nil, errors.WithMessage(err, "no valid channel configuration found")
	}
	return cc, nil
}

// MarshalEtcdRaftMetadata serializes etcd RAFT metadata.
func MarshalEtcdRaftMetadata(md *etcdraft.ConfigMetadata) ([]byte, error) {
	copyMd := proto.Clone(md).(*etcdraft.ConfigMetadata)
	for _, c := range copyMd.Consenters {
		// Expect the user to set the config value for client/server certs to the
		// path where they are persisted locally, then load these files to memory.
		clientCert, err := ioutil.ReadFile(string(c.GetClientTlsCert()))
		if err != nil {
			return nil, fmt.Errorf("cannot load client cert for consenter %s:%d: %s", c.GetHost(), c.GetPort(), err)
		}
		c.ClientTlsCert = clientCert

		serverCert, err := ioutil.ReadFile(string(c.GetServerTlsCert()))
		if err != nil {
			return nil, fmt.Errorf("cannot load server cert for consenter %s:%d: %s", c.GetHost(), c.GetPort(), err)
		}
		c.ServerTlsCert = serverCert
	}
	return proto.Marshal(copyMd)
}
<|endoftext|>


<commit_before>package drive

import (
    "fmt"
    "io"
    "os"
    "time"
    "path/filepath"
    "google.golang.org/api/drive/v3"
)

type DownloadArgs struct {
    Out io.Writer
    Progress io.Writer
    Id string
    Path string
    Force bool
    Recursive bool
    Stdout bool
}

func (self *Drive) Download(args DownloadArgs) error {
    return self.download(args)
}

func (self *Drive) download(args DownloadArgs) error {
    f, err := self.service.Files.Get(args.Id).Fields("id", "name", "size", "mimeType", "md5Checksum").Do()
    if err != nil {
        return fmt.Errorf("Failed to get file: %s", err)
    }

    if isDir(f) && !args.Recursive {
        return fmt.Errorf("'%s' is a directory, use --recursive to download directories", f.Name)
    } else if isDir(f) && args.Recursive {
        return self.downloadDirectory(f, args)
    } else if isBinary(f) {
        return self.downloadBinary(f, args)
    } else if !args.Recursive {
        return fmt.Errorf("'%s' is a google document and must be exported, see the export command", f.Name)
    }

    return nil
}

func (self *Drive) downloadBinary(f *drive.File, args DownloadArgs) error {
    res, err := self.service.Files.Get(f.Id).Download()
    if err != nil {
        return fmt.Errorf("Failed to download file: %s", err)
    }

    // Close body on function exit
    defer res.Body.Close()

    // Wrap response body in progress reader
    srcReader := getProgressReader(res.Body, args.Progress, res.ContentLength)

    if args.Stdout {
        // Write file content to stdout
        _, err := io.Copy(args.Out, srcReader)
        return err
    }

    filename := filepath.Join(args.Path, f.Name)

    // Check if file exists
    if !args.Force && fileExists(filename) {
        return fmt.Errorf("File '%s' already exists, use --force to overwrite", filename)
    }

    // Ensure any parent directories exists
    if err = mkdir(filename); err != nil {
        return err
    }

    // Create new file
    outFile, err := os.Create(filename)
    if err != nil {
        return fmt.Errorf("Unable to create new file: %s", err)
    }

    // Close file on function exit
    defer outFile.Close()

    fmt.Fprintf(args.Out, "\nDownloading %s...\n", f.Name)
    started := time.Now()

    // Save file to disk
    bytes, err := io.Copy(outFile, srcReader)
    if err != nil {
        return fmt.Errorf("Failed saving file: %s", err)
    }

    // Calculate average download rate
    rate := calcRate(f.Size, started, time.Now())

    fmt.Fprintf(args.Out, "Downloaded '%s' at %s/s, total %s\n", filename, formatSize(rate, false), formatSize(bytes, false))

    //if deleteSourceFile {
    //    self.Delete(args.Id)
    //}
    return nil
}

func (self *Drive) downloadDirectory(parent *drive.File, args DownloadArgs) error {
    query := fmt.Sprintf("'%s' in parents", parent.Id)
    fileList, err := self.service.Files.List().Q(query).Fields("files(id,name)").Do()
    if err != nil {
        return fmt.Errorf("Failed listing files: %s", err)
    }

    // Update download path
    path := filepath.Join(args.Path, parent.Name)

    for _, f := range fileList.Files {
        err = self.download(DownloadArgs{
            Out: args.Out,
            Id: f.Id,
            Progress: args.Progress,
            Force: args.Force,
            Path: path,
            Recursive: args.Recursive,
            Stdout: false,
        })

        if err != nil {
            return err
        }
    }

    return nil
}

func isDir(f *drive.File) bool {
    return f.MimeType == DirectoryMimeType
}

func isBinary(f *drive.File) bool {
    return f.Md5Checksum != ""
}
<commit_msg>Copy and update given args<commit_after>package drive

import (
    "fmt"
    "io"
    "os"
    "time"
    "path/filepath"
    "google.golang.org/api/drive/v3"
)

type DownloadArgs struct {
    Out io.Writer
    Progress io.Writer
    Id string
    Path string
    Force bool
    Recursive bool
    Stdout bool
}

func (self *Drive) Download(args DownloadArgs) error {
    return self.download(args)
}

func (self *Drive) download(args DownloadArgs) error {
    f, err := self.service.Files.Get(args.Id).Fields("id", "name", "size", "mimeType", "md5Checksum").Do()
    if err != nil {
        return fmt.Errorf("Failed to get file: %s", err)
    }

    if isDir(f) && !args.Recursive {
        return fmt.Errorf("'%s' is a directory, use --recursive to download directories", f.Name)
    } else if isDir(f) && args.Recursive {
        return self.downloadDirectory(f, args)
    } else if isBinary(f) {
        return self.downloadBinary(f, args)
    } else if !args.Recursive {
        return fmt.Errorf("'%s' is a google document and must be exported, see the export command", f.Name)
    }

    return nil
}

func (self *Drive) downloadBinary(f *drive.File, args DownloadArgs) error {
    res, err := self.service.Files.Get(f.Id).Download()
    if err != nil {
        return fmt.Errorf("Failed to download file: %s", err)
    }

    // Close body on function exit
    defer res.Body.Close()

    // Wrap response body in progress reader
    srcReader := getProgressReader(res.Body, args.Progress, res.ContentLength)

    if args.Stdout {
        // Write file content to stdout
        _, err := io.Copy(args.Out, srcReader)
        return err
    }

    filename := filepath.Join(args.Path, f.Name)

    // Check if file exists
    if !args.Force && fileExists(filename) {
        return fmt.Errorf("File '%s' already exists, use --force to overwrite", filename)
    }

    // Ensure any parent directories exists
    if err = mkdir(filename); err != nil {
        return err
    }

    // Create new file
    outFile, err := os.Create(filename)
    if err != nil {
        return fmt.Errorf("Unable to create new file: %s", err)
    }

    // Close file on function exit
    defer outFile.Close()

    fmt.Fprintf(args.Out, "\nDownloading %s...\n", f.Name)
    started := time.Now()

    // Save file to disk
    bytes, err := io.Copy(outFile, srcReader)
    if err != nil {
        return fmt.Errorf("Failed saving file: %s", err)
    }

    // Calculate average download rate
    rate := calcRate(f.Size, started, time.Now())

    fmt.Fprintf(args.Out, "Downloaded '%s' at %s/s, total %s\n", filename, formatSize(rate, false), formatSize(bytes, false))

    //if deleteSourceFile {
    //    self.Delete(args.Id)
    //}
    return nil
}

func (self *Drive) downloadDirectory(parent *drive.File, args DownloadArgs) error {
    query := fmt.Sprintf("'%s' in parents", parent.Id)
    fileList, err := self.service.Files.List().Q(query).Fields("files(id,name)").Do()
    if err != nil {
        return fmt.Errorf("Failed listing files: %s", err)
    }

    newPath := filepath.Join(args.Path, parent.Name)

    for _, f := range fileList.Files {
        // Copy args and update changed fields
        newArgs := args
        newArgs.Path = newPath
        newArgs.Id = f.Id
        newArgs.Stdout = false

        err = self.download(newArgs)
        if err != nil {
            return err
        }
    }

    return nil
}

func isDir(f *drive.File) bool {
    return f.MimeType == DirectoryMimeType
}

func isBinary(f *drive.File) bool {
    return f.Md5Checksum != ""
}
<|endoftext|>


<commit_before>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package lib

import (
	"context"
	"fmt"
	"net/http"
	"strconv"
	"time"

	monitoring "cloud.google.com/go/monitoring/apiv3/v2"
	"github.com/golang/protobuf/ptypes"
	"github.com/google/go-cmp/cmp"
	"github.com/google/knative-gcp/test/e2e/lib/metrics"
	"google.golang.org/api/iterator"
	monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3"
)

type BrokerMetricAssertion struct {
	ProjectID       string
	BrokerName      string
	BrokerNamespace string
	StartTime       time.Time
	CountPerType    map[string]int64
}

func (a BrokerMetricAssertion) Assert(client *monitoring.MetricClient) error {
	ctx := context.Background()
	start, err := ptypes.TimestampProto(a.StartTime)
	if err != nil {
		return err
	}
	end, err := ptypes.TimestampProto(time.Now())
	if err != nil {
		return err
	}
	it := client.ListTimeSeries(ctx, &monitoringpb.ListTimeSeriesRequest{
		Name:     fmt.Sprintf("projects/%s", a.ProjectID),
		Filter:   a.StackdriverFilter(),
		Interval: &monitoringpb.TimeInterval{StartTime: start, EndTime: end},
		View:     monitoringpb.ListTimeSeriesRequest_FULL,
	})
	gotCount := make(map[string]int64)
	for {
		ts, err := it.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			return err
		}
		labels := ts.GetMetric().GetLabels()
		eventType := labels["event_type"]
		code, err := strconv.Atoi(labels["response_code"])
		if err != nil {
			return fmt.Errorf("metric has invalid response code label: %v", ts.GetMetric())
		}
		if code != http.StatusAccepted {
			return fmt.Errorf("metric has unexpected response code: %v", ts.GetMetric())
		}
		gotCount[eventType] = gotCount[eventType] + metrics.SumCumulative(ts)
	}
	if diff := cmp.Diff(a.CountPerType, gotCount); diff != "" {
		return fmt.Errorf("unexpected broker metric count (-want, +got) = %v", diff)
	}
	return nil
}

func (a BrokerMetricAssertion) StackdriverFilter() string {
	filter := map[string]interface{}{
		"metric.type":                   BrokerEventCountMetricType,
		"resource.type":                 BrokerMetricResourceType,
		"resource.label.namespace_name": a.BrokerNamespace,
		"resource.label.broker_name":    a.BrokerName,
	}
	return metrics.StringifyStackDriverFilter(filter)
}
<commit_msg>Reduce Metric test flakiness caused by sender pod retry sending events (#1661)<commit_after>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package lib

import (
	"context"
	"fmt"
	"net/http"
	"strconv"
	"time"

	monitoring "cloud.google.com/go/monitoring/apiv3/v2"
	"github.com/golang/protobuf/ptypes"
	"github.com/google/go-cmp/cmp"
	"github.com/google/knative-gcp/test/e2e/lib/metrics"
	"google.golang.org/api/iterator"
	monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3"
)

type BrokerMetricAssertion struct {
	ProjectID       string
	BrokerName      string
	BrokerNamespace string
	StartTime       time.Time
	CountPerType    map[string]int64
}

func (a BrokerMetricAssertion) Assert(client *monitoring.MetricClient) error {
	ctx := context.Background()
	start, err := ptypes.TimestampProto(a.StartTime)
	if err != nil {
		return err
	}
	end, err := ptypes.TimestampProto(time.Now())
	if err != nil {
		return err
	}
	it := client.ListTimeSeries(ctx, &monitoringpb.ListTimeSeriesRequest{
		Name:     fmt.Sprintf("projects/%s", a.ProjectID),
		Filter:   a.StackdriverFilter(),
		Interval: &monitoringpb.TimeInterval{StartTime: start, EndTime: end},
		View:     monitoringpb.ListTimeSeriesRequest_FULL,
	})
	gotCount := make(map[string]int64)
	for {
		ts, err := it.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			return err
		}
		labels := ts.GetMetric().GetLabels()
		eventType := labels["event_type"]
		code, err := strconv.Atoi(labels["response_code"])
		if err != nil {
			return fmt.Errorf("metric has invalid response code label: %v", ts.GetMetric())
		}

		// Ignore undesired response code in metric count comparison due to flakiness in broker.
		// The sender pod will retry StatusCode 404, 503 and 500.
		// StatusCode 500 is currently for reducing flakiness caused by Workload Identity credential sync up.
		// We would remove it after https://github.com/google/knative-gcp/issues/1058 lands, as 500 error may indicate bugs in our code.
		if code == http.StatusNotFound || code == http.StatusServiceUnavailable || code == http.StatusInternalServerError {
			continue
		}

		if code != http.StatusAccepted {
			return fmt.Errorf("metric has unexpected response code: %v", ts.GetMetric())
		}
		gotCount[eventType] = gotCount[eventType] + metrics.SumCumulative(ts)
	}
	if diff := cmp.Diff(a.CountPerType, gotCount); diff != "" {
		return fmt.Errorf("unexpected broker metric count (-want, +got) = %v", diff)
	}
	return nil
}

func (a BrokerMetricAssertion) StackdriverFilter() string {
	filter := map[string]interface{}{
		"metric.type":                   BrokerEventCountMetricType,
		"resource.type":                 BrokerMetricResourceType,
		"resource.label.namespace_name": a.BrokerNamespace,
		"resource.label.broker_name":    a.BrokerName,
	}
	return metrics.StringifyStackDriverFilter(filter)
}
<|endoftext|>


<commit_before>package records

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"sync"

	"github.com/mesos/mesos-go/detector"
	_ "github.com/mesos/mesos-go/detector/zoo"
	mesos "github.com/mesos/mesos-go/mesosproto"

	"github.com/mesosphere/mesos-dns/logging"
	"github.com/miekg/dns"
)

// Config holds mesos dns configuration
type Config struct {

	// Mesos master(s): a list of IP:port pairs for one or more Mesos masters
	Masters []string

	// Zookeeper: a single Zk url
	Zk []string

	// Refresh frequency: the frequency in seconds of regenerating records (default 60)
	RefreshSeconds int

	// TTL: the TTL value used for SRV and A records (default 60)
	TTL int

	// Resolver port: port used to listen for slave requests (default 53)
	Port int

	//  Domain: name of the domain used (default "mesos", ie .mesos domain)
	Domain string

	// DNS server: IP address of the DNS server for forwarded accesses
	Resolvers []string

	// Timeout is the default connect/read/write timeout for outbound
	// queries
	Timeout int

	// File is the location of the config.json file
	File string

	// Email is the rname for a SOA
	Email string

	// Mname is the mname for a SOA
	Mname string

	// ListenAddr is the server listener address
	Listener string

	// Leading master info, as identified through Zookeeper
	leader     string
	leaderLock sync.RWMutex
}

// SetConfig instantiates a Config struct read in from config.json
func SetConfig(cjson string) (c Config) {
	c = Config{
		RefreshSeconds: 60,
		TTL:            60,
		Domain:         "mesos",
		Port:           53,
		Timeout:        5,
		Email:          "root.mesos-dns.mesos",
		Resolvers:      []string{"8.8.8.8"},
		Listener:       "0.0.0.0",
		leader:         "",
	}

	usr, _ := user.Current()
	dir := usr.HomeDir + "/"
	cjson = strings.Replace(cjson, "~/", dir, 1)

	path, err := filepath.Abs(cjson)
	if err != nil {
		logging.Error.Println("cannot find configuration file")
		os.Exit(1)
	}

	b, err := ioutil.ReadFile(path)
	if err != nil {
		logging.Error.Println("missing configuration file")
		os.Exit(1)
	}

	err = json.Unmarshal(b, &c)
	if err != nil {
		logging.Error.Println(err)
	}

	if len(c.Resolvers) == 0 {
		c.Resolvers = GetLocalDNS()
	}

	if len(c.Masters) == 0 && len(c.Zk) == 0 {
		logging.Error.Println("please specify mesos masters or zookeeper in config.json")
		os.Exit(1)
	}

	c.Email = strings.Replace(c.Email, "@", ".", -1)
	if c.Email[len(c.Email)-1:] != "." {
		c.Email = c.Email + "."
	}

	c.Domain = strings.ToLower(c.Domain)
	c.Mname = "mesos-dns." + c.Domain + "."

	logging.Verbose.Println("Mesos-DNS configuration:")
	if len(c.Zk) == 0 {
		logging.Verbose.Println("   - Masters: " + strings.Join(c.Masters, ", "))
	} else {
		logging.Verbose.Println("   - Zookeeper: " + strings.Join(c.Zk, ", "))
	}
	logging.Verbose.Println("   - RefreshSeconds: ", c.RefreshSeconds)
	logging.Verbose.Println("   - TTL: ", c.TTL)
	logging.Verbose.Println("   - Domain: " + c.Domain)
	logging.Verbose.Println("   - Port: ", c.Port)
	logging.Verbose.Println("   - Timeout: ", c.Timeout)
	logging.Verbose.Println("   - Listener: " + c.Listener)
	logging.Verbose.Println("   - Resolvers: " + strings.Join(c.Resolvers, ", "))
	logging.Verbose.Println("   - Email: " + c.Email)
	logging.Verbose.Println("   - Mname: " + c.Mname)

	return c
}

// localAddies returns an array of local ipv4 addresses
func localAddies() []string {
	addies, err := net.InterfaceAddrs()
	if err != nil {
		logging.Error.Println(err)
	}

	bad := []string{}

	for i := 0; i < len(addies); i++ {
		ip, _, err := net.ParseCIDR(addies[i].String())
		if err != nil {
			logging.Error.Println(err)
		}
		t4 := ip.To4()
		if t4 != nil {
			bad = append(bad, t4.String())
		}
	}

	return bad
}

// nonLocalAddies only returns non-local ns entries
func nonLocalAddies(cservers []string) []string {
	bad := localAddies()

	good := []string{}

	for i := 0; i < len(cservers); i++ {
		local := false
		for x := 0; x < len(bad); x++ {
			if cservers[i] == bad[x] {
				local = true
			}
		}

		if !local {
			good = append(good, cservers[i])
		}
	}

	return good
}

// GetLocalDNS returns the first nameserver in /etc/resolv.conf
// used for out of mesos domain queries
func GetLocalDNS() []string {
	conf, err := dns.ClientConfigFromFile("/etc/resolv.conf")
	if err != nil {
		logging.Error.Println(err)
		os.Exit(2)
	}

	return nonLocalAddies(conf.Servers)
}

// Start a Zookeeper listener to track leading master
func ZKdetect(c Config) {

	// start listener
	logging.Verbose.Println("Starting master detector for ZK ", c.Zk[0])
	md, err := detector.New(c.Zk[0])
	if err != nil {
		logging.Error.Println("failed to create master detector: ", err)
		os.Exit(1)
	}

	// and listen for master changes
	if err := md.Detect(detector.OnMasterChanged(func(info *mesos.MasterInfo) {
		// making this tomic
		c.leaderLock.Lock()
		defer c.leaderLock.Unlock()
		if info == nil {
			c.leader = ""
			logging.Error.Println("No leader available in Zookeeper.")

		} else if host := info.GetHostname(); host != "" {
			c.leader = host
		} else {
			// unpack IPv4
			octets := make([]byte, 4, 4)
			binary.BigEndian.PutUint32(octets, info.GetIp())
			ipv4 := net.IP(octets)
			c.leader = ipv4.String()
		}
		if len(c.leader) > 0 {
			c.leader = fmt.Sprintf("%s:%d", c.leader, info.GetPort())
		}
		logging.Verbose.Println("New master in Zookeeper ", c.leader)
	})); err != nil {
		logging.Error.Println("failed to initialize master detector ", err)
		os.Exit(1)
	}

}
<commit_msg>minor fix for logging.<commit_after>package records

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"sync"

	"github.com/mesos/mesos-go/detector"
	_ "github.com/mesos/mesos-go/detector/zoo"
	mesos "github.com/mesos/mesos-go/mesosproto"

	"github.com/mesosphere/mesos-dns/logging"
	"github.com/miekg/dns"
)

// Config holds mesos dns configuration
type Config struct {

	// Mesos master(s): a list of IP:port pairs for one or more Mesos masters
	Masters []string

	// Zookeeper: a single Zk url
	Zk []string

	// Refresh frequency: the frequency in seconds of regenerating records (default 60)
	RefreshSeconds int

	// TTL: the TTL value used for SRV and A records (default 60)
	TTL int

	// Resolver port: port used to listen for slave requests (default 53)
	Port int

	//  Domain: name of the domain used (default "mesos", ie .mesos domain)
	Domain string

	// DNS server: IP address of the DNS server for forwarded accesses
	Resolvers []string

	// Timeout is the default connect/read/write timeout for outbound
	// queries
	Timeout int

	// File is the location of the config.json file
	File string

	// Email is the rname for a SOA
	Email string

	// Mname is the mname for a SOA
	Mname string

	// ListenAddr is the server listener address
	Listener string

	// Leading master info, as identified through Zookeeper
	leader     string
	leaderLock sync.RWMutex
}

// SetConfig instantiates a Config struct read in from config.json
func SetConfig(cjson string) (c Config) {
	c = Config{
		RefreshSeconds: 60,
		TTL:            60,
		Domain:         "mesos",
		Port:           53,
		Timeout:        5,
		Email:          "root.mesos-dns.mesos",
		Resolvers:      []string{"8.8.8.8"},
		Listener:       "0.0.0.0",
		leader:         "",
	}

	usr, _ := user.Current()
	dir := usr.HomeDir + "/"
	cjson = strings.Replace(cjson, "~/", dir, 1)

	path, err := filepath.Abs(cjson)
	if err != nil {
		logging.Error.Println("cannot find configuration file")
		os.Exit(1)
	}

	b, err := ioutil.ReadFile(path)
	if err != nil {
		logging.Error.Println("missing configuration file")
		os.Exit(1)
	}

	err = json.Unmarshal(b, &c)
	if err != nil {
		logging.Error.Println(err)
	}

	if len(c.Resolvers) == 0 {
		c.Resolvers = GetLocalDNS()
	}

	if len(c.Masters) == 0 && len(c.Zk) == 0 {
		logging.Error.Println("please specify mesos masters or zookeeper in config.json")
		os.Exit(1)
	}

	c.Email = strings.Replace(c.Email, "@", ".", -1)
	if c.Email[len(c.Email)-1:] != "." {
		c.Email = c.Email + "."
	}

	c.Domain = strings.ToLower(c.Domain)
	c.Mname = "mesos-dns." + c.Domain + "."

	logging.Verbose.Println("Mesos-DNS configuration:")
	if len(c.Masters) != 0 {
		logging.Verbose.Println("   - Masters: " + strings.Join(c.Masters, ", "))
	}
	if len(c.Zk) != 0 {
		logging.Verbose.Println("   - Zookeeper: " + strings.Join(c.Zk, ", "))
	}
	logging.Verbose.Println("   - RefreshSeconds: ", c.RefreshSeconds)
	logging.Verbose.Println("   - TTL: ", c.TTL)
	logging.Verbose.Println("   - Domain: " + c.Domain)
	logging.Verbose.Println("   - Port: ", c.Port)
	logging.Verbose.Println("   - Timeout: ", c.Timeout)
	logging.Verbose.Println("   - Listener: " + c.Listener)
	logging.Verbose.Println("   - Resolvers: " + strings.Join(c.Resolvers, ", "))
	logging.Verbose.Println("   - Email: " + c.Email)
	logging.Verbose.Println("   - Mname: " + c.Mname)

	return c
}

// localAddies returns an array of local ipv4 addresses
func localAddies() []string {
	addies, err := net.InterfaceAddrs()
	if err != nil {
		logging.Error.Println(err)
	}

	bad := []string{}

	for i := 0; i < len(addies); i++ {
		ip, _, err := net.ParseCIDR(addies[i].String())
		if err != nil {
			logging.Error.Println(err)
		}
		t4 := ip.To4()
		if t4 != nil {
			bad = append(bad, t4.String())
		}
	}

	return bad
}

// nonLocalAddies only returns non-local ns entries
func nonLocalAddies(cservers []string) []string {
	bad := localAddies()

	good := []string{}

	for i := 0; i < len(cservers); i++ {
		local := false
		for x := 0; x < len(bad); x++ {
			if cservers[i] == bad[x] {
				local = true
			}
		}

		if !local {
			good = append(good, cservers[i])
		}
	}

	return good
}

// GetLocalDNS returns the first nameserver in /etc/resolv.conf
// used for out of mesos domain queries
func GetLocalDNS() []string {
	conf, err := dns.ClientConfigFromFile("/etc/resolv.conf")
	if err != nil {
		logging.Error.Println(err)
		os.Exit(2)
	}

	return nonLocalAddies(conf.Servers)
}

// Start a Zookeeper listener to track leading master
func ZKdetect(c Config) {

	// start listener
	logging.Verbose.Println("Starting master detector for ZK ", c.Zk[0])
	md, err := detector.New(c.Zk[0])
	if err != nil {
		logging.Error.Println("failed to create master detector: ", err)
		os.Exit(1)
	}

	// and listen for master changes
	if err := md.Detect(detector.OnMasterChanged(func(info *mesos.MasterInfo) {
		// making this tomic
		c.leaderLock.Lock()
		defer c.leaderLock.Unlock()
		if info == nil {
			c.leader = ""
			logging.Error.Println("No leader available in Zookeeper.")

		} else if host := info.GetHostname(); host != "" {
			c.leader = host
		} else {
			// unpack IPv4
			octets := make([]byte, 4, 4)
			binary.BigEndian.PutUint32(octets, info.GetIp())
			ipv4 := net.IP(octets)
			c.leader = ipv4.String()
		}
		if len(c.leader) > 0 {
			c.leader = fmt.Sprintf("%s:%d", c.leader, info.GetPort())
		}
		logging.Verbose.Println("New master in Zookeeper ", c.leader)
	})); err != nil {
		logging.Error.Println("failed to initialize master detector ", err)
		os.Exit(1)
	}

}
<|endoftext|>


<commit_before>package piglatin

import (
	. "github.com/franela/goblin"
	"testing"
)

func Test(t *testing.T) {
	g := Goblin(t)

	VerifyTranslation := func(word, expected string) {
		g.Assert(Translate(word)).Equal(expected)
	}

	g.Describe("Translations", func() {
		g.Describe("First character conditions", func() {
			g.It("starts with a consonant", func() {
				VerifyTranslation("derp", "erpday")
			})

			g.It("starts with a vowel", func() {
				VerifyTranslation("eggnog", "eggnogway")
			})

			g.It("starts with a consonant chunk", func() {
				VerifyTranslation("chloroform", "oroformchlay")
			})
		})

		g.Describe("Ends in punctuation", func() {
			g.It("ends with punctuation", func() {
				VerifyTranslation("herp,", "erphay,")
			})

			g.It("ends with a period", func() {
				VerifyTranslation("derp.", "erpday.")
			})
		})
	})
}
<commit_msg>TIL . on imports is bad practice<commit_after>package piglatin

import (
	"github.com/franela/goblin"
	"testing"
)

func Test(t *testing.T) {
	g := goblin.Goblin(t)

	VerifyTranslation := func(word, expected string) {
		g.Assert(Translate(word)).Equal(expected)
	}

	g.Describe("Translations", func() {
		g.Describe("First character conditions", func() {
			g.It("starts with a consonant", func() {
				VerifyTranslation("derp", "erpday")
			})

			g.It("starts with a vowel", func() {
				VerifyTranslation("eggnog", "eggnogway")
			})

			g.It("starts with a consonant chunk", func() {
				VerifyTranslation("chloroform", "oroformchlay")
			})
		})

		g.Describe("Ends in punctuation", func() {
			g.It("ends with punctuation", func() {
				VerifyTranslation("herp,", "erphay,")
			})

			g.It("ends with a period", func() {
				VerifyTranslation("derp.", "erpday.")
			})
		})
	})
}
<|endoftext|>


<commit_before>package email

import (
	"testing"

	"bytes"
	"crypto/rand"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/mail"
	"net/smtp"
	"os"
)

func TestEmailTextHtmlAttachment(t *testing.T) {
	e := NewEmail()
	e.From = "Jordan Wright <test@example.com>"
	e.To = []string{"test@example.com"}
	e.Bcc = []string{"test_bcc@example.com"}
	e.Cc = []string{"test_cc@example.com"}
	e.Subject = "Awesome Subject"
	e.Text = []byte("Text Body is, of course, supported!\n")
	e.HTML = []byte("
Fancy Html is supported, too!
\n")
	e.Attach(bytes.NewBufferString("Rad attachement"), "rad.txt", "text/plain; charset=utf-8")

	raw, err := e.Bytes()
	if err != nil {
		t.Fatal("Failed to render message: ", e)
	}
	ioutil.WriteFile("golden", raw, os.ModePerm)

	msg, err := mail.ReadMessage(bytes.NewBuffer(raw))
	if err != nil {
		t.Fatal("Could not parse rendered message: ", err)
	}

	expectedHeaders := map[string]string{
		"To":      "test@example.com",
		"From":    "Jordan Wright <test@example.com>",
		"Cc":      "test_cc@example.com",
		"Subject": "Awesome Subject",
	}

	for header, expected := range expectedHeaders {
		if val := msg.Header.Get(header); val != expected {
			t.Errorf("Wrong value for message header %s: %v != %v", header, expected, val)
		}
	}

	// Were the right headers set?
	ct := msg.Header.Get("Content-type")
	mt, params, err := mime.ParseMediaType(ct)
	if err != nil {
		t.Fatal("Content-type header is invalid: ", ct)
	} else if mt != "multipart/mixed" {
		t.Fatalf("Content-type expected \"multipart/mixed\", not %v", mt)
	}
	b := params["boundary"]
	if b == "" {
		t.Fatalf("Invalid or missing boundary parameter: ", b)
	}
	if len(params) != 1 {
		t.Fatal("Unexpected content-type parameters")
	}

	// Is the generated message parsable?
	mixed := multipart.NewReader(msg.Body, params["boundary"])

	text, err := mixed.NextPart()
	if err != nil {
		t.Fatalf("Could not find text component of email: ", err)
	}

	// Does the text portion match what we expect?
	mt, params, err = mime.ParseMediaType(text.Header.Get("Content-type"))
	if err != nil {
		t.Fatal("Could not parse message's Content-Type")
	} else if mt != "multipart/alternative" {
		t.Fatal("Message missing multipart/alternative")
	}
	mpReader := multipart.NewReader(text, params["boundary"])
	part, err := mpReader.NextPart()
	if err != nil {
		t.Fatal("Could not read plain text component of message: ", err)
	}
	plainText, err := ioutil.ReadAll(part)
	if err != nil {
		t.Fatal("Could not read plain text component of message: ", err)
	}
	if !bytes.Equal(plainText, []byte("Text Body is, of course, supported!\r\n")) {
		t.Fatalf("Plain text is broken: %#q", plainText)
	}

	// Check attachments.
	_, err = mixed.NextPart()
	if err != nil {
		t.Fatalf("Could not find attachemnt compoenent of email: ", err)
	}

	if _, err = mixed.NextPart(); err != io.EOF {
		t.Error("Expected only text and one attachement!")
	}

}

func ExampleGmail() {
	e := NewEmail()
	e.From = "Jordan Wright <test@gmail.com>"
	e.To = []string{"test@example.com"}
	e.Bcc = []string{"test_bcc@example.com"}
	e.Cc = []string{"test_cc@example.com"}
	e.Subject = "Awesome Subject"
	e.Text = []byte("Text Body is, of course, supported!\n")
	e.HTML = []byte("
Fancy Html is supported, too!
\n")
	e.Send("smtp.gmail.com:587", smtp.PlainAuth("", e.From, "password123", "smtp.gmail.com"))
}

func ExampleAttach() {
	e := NewEmail()
	e.AttachFile("test.txt")
}

func Test_base64Wrap(t *testing.T) {
	file := "I'm a file long enough to force the function to wrap a\n" +
		"couple of lines, but I stop short of the end of one line and\n" +
		"have some padding dangling at the end."
	encoded := "SSdtIGEgZmlsZSBsb25nIGVub3VnaCB0byBmb3JjZSB0aGUgZnVuY3Rpb24gdG8gd3JhcCBhCmNv\r\n" +
		"dXBsZSBvZiBsaW5lcywgYnV0IEkgc3RvcCBzaG9ydCBvZiB0aGUgZW5kIG9mIG9uZSBsaW5lIGFu\r\n" +
		"ZApoYXZlIHNvbWUgcGFkZGluZyBkYW5nbGluZyBhdCB0aGUgZW5kLg==\r\n"

	var buf bytes.Buffer
	base64Wrap(&buf, []byte(file))
	if !bytes.Equal(buf.Bytes(), []byte(encoded)) {
		t.Fatalf("Encoded file does not match expected: %#q != %#q", string(buf.Bytes()), encoded)
	}
}

func Test_quotedPrintEncode(t *testing.T) {
	var buf bytes.Buffer
	text := []byte("Dear reader!\n\n" +
		"This is a test email to try and capture some of the corner cases that exist within\n" +
		"the quoted-printable encoding.\n" +
		"There are some wacky parts like =, and this input assumes UNIX line breaks so\r\n" +
		"it can come out a little weird.  Also, we need to support unicode so here's a fish: 🐟\n")
	expected := []byte("Dear reader!\r\n\r\n" +
		"This is a test email to try and capture some of the corner cases that exist=\r\n" +
		" within\r\n" +
		"the quoted-printable encoding.\r\n" +
		"There are some wacky parts like =3D, and this input assumes UNIX line break=\r\n" +
		"s so=0D\r\n" +
		"it can come out a little weird.  Also, we need to support unicode so here's=\r\n" +
		" a fish: =F0=9F=90=9F\r\n")

	if err := quotePrintEncode(&buf, text); err != nil {
		t.Fatal("quotePrintEncode: ", err)
	}

	if b := buf.Bytes(); !bytes.Equal(b, expected) {
		t.Errorf("quotedPrintEncode generated incorrect results: %#q != %#q", b, expected)
	}
}

func Benchmark_quotedPrintEncode(b *testing.B) {
	text := []byte("Dear reader!\n\n" +
		"This is a test email to try and capture some of the corner cases that exist within\n" +
		"the quoted-printable encoding.\n" +
		"There are some wacky parts like =, and this input assumes UNIX line breaks so\r\n" +
		"it can come out a little weird.  Also, we need to support unicode so here's a fish: 🐟\n")

	for i := 0; i <= b.N; i++ {
		if err := quotePrintEncode(ioutil.Discard, text); err != nil {
			panic(err)
		}
	}
}

func Benchmark_base64Wrap(b *testing.B) {
	// Reasonable base case; 128K random bytes
	file := make([]byte, 128*1024)
	if _, err := rand.Read(file); err != nil {
		panic(err)
	}
	for i := 0; i <= b.N; i++ {
		base64Wrap(ioutil.Discard, file)
	}
}
<commit_msg>Remove tinker cruft from test<commit_after>package email

import (
	"testing"

	"bytes"
	"crypto/rand"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/mail"
	"net/smtp"
	"os"
)

func TestEmailTextHtmlAttachment(t *testing.T) {
	e := NewEmail()
	e.From = "Jordan Wright <test@example.com>"
	e.To = []string{"test@example.com"}
	e.Bcc = []string{"test_bcc@example.com"}
	e.Cc = []string{"test_cc@example.com"}
	e.Subject = "Awesome Subject"
	e.Text = []byte("Text Body is, of course, supported!\n")
	e.HTML = []byte("
Fancy Html is supported, too!
\n")
	e.Attach(bytes.NewBufferString("Rad attachement"), "rad.txt", "text/plain; charset=utf-8")

	raw, err := e.Bytes()
	if err != nil {
		t.Fatal("Failed to render message: ", e)
	}

	msg, err := mail.ReadMessage(bytes.NewBuffer(raw))
	if err != nil {
		t.Fatal("Could not parse rendered message: ", err)
	}

	expectedHeaders := map[string]string{
		"To":      "test@example.com",
		"From":    "Jordan Wright <test@example.com>",
		"Cc":      "test_cc@example.com",
		"Subject": "Awesome Subject",
	}

	for header, expected := range expectedHeaders {
		if val := msg.Header.Get(header); val != expected {
			t.Errorf("Wrong value for message header %s: %v != %v", header, expected, val)
		}
	}

	// Were the right headers set?
	ct := msg.Header.Get("Content-type")
	mt, params, err := mime.ParseMediaType(ct)
	if err != nil {
		t.Fatal("Content-type header is invalid: ", ct)
	} else if mt != "multipart/mixed" {
		t.Fatalf("Content-type expected \"multipart/mixed\", not %v", mt)
	}
	b := params["boundary"]
	if b == "" {
		t.Fatalf("Invalid or missing boundary parameter: ", b)
	}
	if len(params) != 1 {
		t.Fatal("Unexpected content-type parameters")
	}

	// Is the generated message parsable?
	mixed := multipart.NewReader(msg.Body, params["boundary"])

	text, err := mixed.NextPart()
	if err != nil {
		t.Fatalf("Could not find text component of email: ", err)
	}

	// Does the text portion match what we expect?
	mt, params, err = mime.ParseMediaType(text.Header.Get("Content-type"))
	if err != nil {
		t.Fatal("Could not parse message's Content-Type")
	} else if mt != "multipart/alternative" {
		t.Fatal("Message missing multipart/alternative")
	}
	mpReader := multipart.NewReader(text, params["boundary"])
	part, err := mpReader.NextPart()
	if err != nil {
		t.Fatal("Could not read plain text component of message: ", err)
	}
	plainText, err := ioutil.ReadAll(part)
	if err != nil {
		t.Fatal("Could not read plain text component of message: ", err)
	}
	if !bytes.Equal(plainText, []byte("Text Body is, of course, supported!\r\n")) {
		t.Fatalf("Plain text is broken: %#q", plainText)
	}

	// Check attachments.
	_, err = mixed.NextPart()
	if err != nil {
		t.Fatalf("Could not find attachemnt compoenent of email: ", err)
	}

	if _, err = mixed.NextPart(); err != io.EOF {
		t.Error("Expected only text and one attachement!")
	}

}

func ExampleGmail() {
	e := NewEmail()
	e.From = "Jordan Wright <test@gmail.com>"
	e.To = []string{"test@example.com"}
	e.Bcc = []string{"test_bcc@example.com"}
	e.Cc = []string{"test_cc@example.com"}
	e.Subject = "Awesome Subject"
	e.Text = []byte("Text Body is, of course, supported!\n")
	e.HTML = []byte("
Fancy Html is supported, too!
\n")
	e.Send("smtp.gmail.com:587", smtp.PlainAuth("", e.From, "password123", "smtp.gmail.com"))
}

func ExampleAttach() {
	e := NewEmail()
	e.AttachFile("test.txt")
}

func Test_base64Wrap(t *testing.T) {
	file := "I'm a file long enough to force the function to wrap a\n" +
		"couple of lines, but I stop short of the end of one line and\n" +
		"have some padding dangling at the end."
	encoded := "SSdtIGEgZmlsZSBsb25nIGVub3VnaCB0byBmb3JjZSB0aGUgZnVuY3Rpb24gdG8gd3JhcCBhCmNv\r\n" +
		"dXBsZSBvZiBsaW5lcywgYnV0IEkgc3RvcCBzaG9ydCBvZiB0aGUgZW5kIG9mIG9uZSBsaW5lIGFu\r\n" +
		"ZApoYXZlIHNvbWUgcGFkZGluZyBkYW5nbGluZyBhdCB0aGUgZW5kLg==\r\n"

	var buf bytes.Buffer
	base64Wrap(&buf, []byte(file))
	if !bytes.Equal(buf.Bytes(), []byte(encoded)) {
		t.Fatalf("Encoded file does not match expected: %#q != %#q", string(buf.Bytes()), encoded)
	}
}

func Test_quotedPrintEncode(t *testing.T) {
	var buf bytes.Buffer
	text := []byte("Dear reader!\n\n" +
		"This is a test email to try and capture some of the corner cases that exist within\n" +
		"the quoted-printable encoding.\n" +
		"There are some wacky parts like =, and this input assumes UNIX line breaks so\r\n" +
		"it can come out a little weird.  Also, we need to support unicode so here's a fish: 🐟\n")
	expected := []byte("Dear reader!\r\n\r\n" +
		"This is a test email to try and capture some of the corner cases that exist=\r\n" +
		" within\r\n" +
		"the quoted-printable encoding.\r\n" +
		"There are some wacky parts like =3D, and this input assumes UNIX line break=\r\n" +
		"s so=0D\r\n" +
		"it can come out a little weird.  Also, we need to support unicode so here's=\r\n" +
		" a fish: =F0=9F=90=9F\r\n")

	if err := quotePrintEncode(&buf, text); err != nil {
		t.Fatal("quotePrintEncode: ", err)
	}

	if b := buf.Bytes(); !bytes.Equal(b, expected) {
		t.Errorf("quotedPrintEncode generated incorrect results: %#q != %#q", b, expected)
	}
}

func Benchmark_quotedPrintEncode(b *testing.B) {
	text := []byte("Dear reader!\n\n" +
		"This is a test email to try and capture some of the corner cases that exist within\n" +
		"the quoted-printable encoding.\n" +
		"There are some wacky parts like =, and this input assumes UNIX line breaks so\r\n" +
		"it can come out a little weird.  Also, we need to support unicode so here's a fish: 🐟\n")

	for i := 0; i <= b.N; i++ {
		if err := quotePrintEncode(ioutil.Discard, text); err != nil {
			panic(err)
		}
	}
}

func Benchmark_base64Wrap(b *testing.B) {
	// Reasonable base case; 128K random bytes
	file := make([]byte, 128*1024)
	if _, err := rand.Read(file); err != nil {
		panic(err)
	}
	for i := 0; i <= b.N; i++ {
		base64Wrap(ioutil.Discard, file)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os/signal"
	"syscall"
	"os"
)

func main()  {
	var nodeName string
	var port = flag.Int("port", 8090, "Port to bind to on the localhost interface")
	flag.StringVar(&nodeName,"name", "my", "Name of the running instance")
	flag.Parse()

	router := NewRouter(nodeName)
	log.Printf("Starting a server on localhost:%d", *port)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), router))
}

func init()  {
	LoadConfig()

	sc := make(chan os.Signal, 1)

	signal.Notify(sc, syscall.SIGHUP)

	go func ()  {
		for {
			<-sc
			fox.LoadConfig()
		}
	}()
}<commit_msg>FoxService import standardization<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os/signal"
	"syscall"
	"os"
	. "fox"
)

func main()  {
	var nodeName string
	var port = flag.Int("port", 8090, "Port to bind to on the localhost interface")
	flag.StringVar(&nodeName,"name", "my", "Name of the running instance")
	flag.Parse()

	router := NewRouter(nodeName)
	log.Printf("Starting a server on localhost:%d", *port)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), router))
}

func init()  {
	LoadConfig()

	sc := make(chan os.Signal, 1)

	signal.Notify(sc, syscall.SIGHUP)

	go func ()  {
		for {
			<-sc
			LoadConfig()
		}
	}()
}<|endoftext|>


<commit_before>// Generates .travis.yml configuration using pkg/goversion/compat.go
// Usage go run scripts/gen-travis.go > .travis.yml

package main

import (
	"bufio"
	"fmt"
	"os"
	"text/template"

	"github.com/go-delve/delve/pkg/goversion"
)

type arguments struct {
	GoVersions []goVersion
}

type goVersion struct {
	Major, Minor int
}

var maxVersion = goVersion{Major: goversion.MaxSupportedVersionOfGoMajor, Minor: goversion.MaxSupportedVersionOfGoMinor}
var minVersion = goVersion{Major: goversion.MinSupportedVersionOfGoMajor, Minor: goversion.MinSupportedVersionOfGoMinor}

func (v goVersion) dec() goVersion {
	v.Minor--
	if v.Minor < 0 {
		panic("TODO: fill the maximum minor version number for v.Maxjor here")
	}
	return v
}

func (v goVersion) MaxVersion() bool {
	return v == maxVersion
}

func (v goVersion) DotX() string {
	return fmt.Sprintf("%d.%d.x", v.Major, v.Minor)
}

func (v goVersion) String() string {
	return fmt.Sprintf("%d.%d", v.Major, v.Minor)
}

func main() {
	var args arguments

	args.GoVersions = append(args.GoVersions, maxVersion)
	for {
		v := args.GoVersions[len(args.GoVersions)-1].dec()
		args.GoVersions = append(args.GoVersions, v)
		if v == minVersion {
			break
		}
	}

	out := bufio.NewWriter(os.Stdout)
	err := template.Must(template.New("travis.yml").Parse(`language: go
sudo: required

os:
  - linux
  - osx
  - windows

arch:
  - amd64
  - arm64

go:
{{- range .GoVersions}}
  - {{.DotX}}
{{- end}}
  - tip

matrix:
  allow_failures:
    - go: tip
  exclude:
    - os: osx
      arch: arm64
    - os: windows
      arch: arm64
{{- /* Exclude all testing on anything except the most recent version of Go for anything that isn't (GOOS=linux, GOARCH=amd64)*/ -}}
{{- range .GoVersions}}{{if not .MaxVersion}}
    - os: windows
      go: {{.DotX}}
{{- end}}{{end -}}
{{- range .GoVersions}}{{if not .MaxVersion}}
    - os: osx
      go: {{.DotX}}
{{- end}}{{end -}}
{{- range .GoVersions}}{{if not .MaxVersion}}
    - arch: arm64
      go: {{.DotX}}
{{- end}}{{end}}
    - os: windows
      go: tip
    - arch: arm64
      go: tip

before_install:
  - export GOFLAGS=-mod=vendor
  - if [ $TRAVIS_OS_NAME = "linux" ]; then sudo apt-get -qq update; sudo apt-get install -y dwz; echo "dwz version $(dwz --version)"; fi
  - if [ $TRAVIS_OS_NAME = "windows" ]; then choco install procdump make; fi

# 386 linux
jobs:
  include:
    -  os: linux
       services: docker
       env: go_32_version={{index .GoVersions 0}}

script: >-
    if [ $TRAVIS_OS_NAME = "linux" ] && [ $go_32_version ]; then
      docker pull i386/centos:7;
      docker run -v $(pwd):/delve --privileged i386/centos:7 /bin/bash -c "set -x && \
           cd delve && \
           yum -y update && yum -y upgrade && \
           yum -y install wget make git gcc && \
           wget -q https://dl.google.com/go/go${go_32_version}.linux-386.tar.gz && \
           tar -C /usr/local -xzf go${go_32_version}.linux-386.tar.gz && \
           export PATH=$PATH:/usr/local/go/bin && \
           go version && \
           uname -a && \
           make test";
    else
      make test;
    fi

cache:
  directories:
    - $HOME/AppData/Local/Temp/chocolatey
`)).Execute(out, args)

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error executing template: %v", err)
		os.Exit(1)
	}
	_ = out.Flush()
}
<commit_msg>scripts: Update gen-travis.go<commit_after>// Generates .travis.yml configuration using pkg/goversion/compat.go
// Usage go run scripts/gen-travis.go > .travis.yml

package main

import (
	"bufio"
	"fmt"
	"os"
	"text/template"

	"github.com/go-delve/delve/pkg/goversion"
)

type arguments struct {
	GoVersions []goVersion
}

type goVersion struct {
	Major, Minor int
}

var maxVersion = goVersion{Major: goversion.MaxSupportedVersionOfGoMajor, Minor: goversion.MaxSupportedVersionOfGoMinor}
var minVersion = goVersion{Major: goversion.MinSupportedVersionOfGoMajor, Minor: goversion.MinSupportedVersionOfGoMinor}

func (v goVersion) dec() goVersion {
	v.Minor--
	if v.Minor < 0 {
		panic("TODO: fill the maximum minor version number for v.Maxjor here")
	}
	return v
}

func (v goVersion) MaxVersion() bool {
	return v == maxVersion
}

func (v goVersion) DotX() string {
	return fmt.Sprintf("%d.%d.x", v.Major, v.Minor)
}

func (v goVersion) String() string {
	return fmt.Sprintf("%d.%d", v.Major, v.Minor)
}

func main() {
	var args arguments

	args.GoVersions = append(args.GoVersions, maxVersion)
	for {
		v := args.GoVersions[len(args.GoVersions)-1].dec()
		args.GoVersions = append(args.GoVersions, v)
		if v == minVersion {
			break
		}
	}

	out := bufio.NewWriter(os.Stdout)
	err := template.Must(template.New("travis.yml").Parse(`language: go
sudo: required
go_import_path: github.com/go-delve/delve

os:
  - linux
  - osx
  - windows

arch:
  - amd64
  - arm64

go:
{{- range .GoVersions}}
  - {{.DotX}}
{{- end}}
  - tip

matrix:
  allow_failures:
    - go: tip
  exclude:
    - os: osx
      arch: arm64
    - os: windows
      arch: arm64
{{- /* Exclude all testing on anything except the most recent version of Go for anything that isn't (GOOS=linux, GOARCH=amd64)*/ -}}
{{- range .GoVersions}}{{if not .MaxVersion}}
    - os: windows
      go: {{.DotX}}
{{- end}}{{end -}}
{{- range .GoVersions}}{{if not .MaxVersion}}
    - os: osx
      go: {{.DotX}}
{{- end}}{{end -}}
{{- range .GoVersions}}{{if not .MaxVersion}}
    - arch: arm64
      go: {{.DotX}}
{{- end}}{{end}}
    - os: windows
      go: tip
    - arch: arm64
      go: tip

before_install:
  - export GOFLAGS=-mod=vendor
  - if [ $TRAVIS_OS_NAME = "linux" ]; then sudo apt-get -qq update; sudo apt-get install -y dwz; echo "dwz version $(dwz --version)"; fi
  - if [ $TRAVIS_OS_NAME = "windows" ]; then choco install procdump make; fi

# 386 linux
jobs:
  include:
    -  os: linux
       services: docker
       env: go_32_version={{index .GoVersions 0}}

script: >-
    if [ $TRAVIS_OS_NAME = "linux" ] && [ $go_32_version ]; then
      docker pull i386/centos:7;
      docker run -v $(pwd):/delve --privileged i386/centos:7 /bin/bash -c "set -x && \
           cd delve && \
           yum -y update && yum -y upgrade && \
           yum -y install wget make git gcc && \
           wget -q https://dl.google.com/go/go${go_32_version}.linux-386.tar.gz && \
           tar -C /usr/local -xzf go${go_32_version}.linux-386.tar.gz && \
           export PATH=$PATH:/usr/local/go/bin && \
           go version && \
           uname -a && \
           make test";
    else
      make test;
    fi

cache:
  directories:
    - $HOME/AppData/Local/Temp/chocolatey
`)).Execute(out, args)

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error executing template: %v", err)
		os.Exit(1)
	}
	_ = out.Flush()
}
<|endoftext|>


<commit_before>package app

import (
	"strings"

	"github.com/materials-commons/config"
	"github.com/materials-commons/gohandy/ezhttp"
)

type mcapi struct{}

var MCApi mcapi

func (a mcapi) MCUrl() string {
	return config.GetString("mcurl")
}

func (a mcapi) MCClient() *ezhttp.EzClient {
	mcurl := a.MCUrl()
	if strings.HasPrefix(mcurl, "https") {
		return ezhttp.NewSSLClient()
	}
	return ezhttp.NewClient()
}
<commit_msg>Add utility functions for dealing with the api.<commit_after>package app

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"

	"github.com/gtarcea/1DevDayTalk2014/app"
	"github.com/materials-commons/config"
	"github.com/materials-commons/gohandy/ezhttp"
	"gnd.la/net/urlutil"
)

type mcapi struct{}

var MCApi mcapi

func (a mcapi) MCUrl() string {
	return config.GetString("mcurl")
}

func (a mcapi) MCClient() *ezhttp.EzClient {
	mcurl := a.MCUrl()
	if strings.HasPrefix(mcurl, "https") {
		return ezhttp.NewSSLClient()
	}
	return ezhttp.NewClient()
}

func (a mcapi) APIUrl(path string) string {
	values := url.Values{}
	values.Add("apikey", config.GetString("apikey"))
	mcurl := urlutil.MustJoin(a.MCUrl(), path)
	mcurl = urlutil.AppendQuery(mcurl, values)
	return mcurl
}

func (a mcapi) APIError(resp *http.Response, errs []error) error {
	switch {
	case len(errs) != 0:
		return app.ErrInvalid
	case resp.StatusCode > 299:
		return fmt.Errorf("HTTP Error: %s", resp.Status)
	default:
		return nil
	}
}

func (a mcapi) ToJSON(from string, to interface{}) error {
	err := json.Unmarshal([]byte(from), to)
	return err
}
<|endoftext|>


<commit_before>package ice

import (
	"fmt"
	"math/rand"
	"net"
	"sync"
	"time"

	"github.com/pions/pkg/stun"
	"github.com/pions/webrtc/internal/util"
	"github.com/pkg/errors"
)

const Unknown = iota

// OutboundCallback is the user defined Callback that is called when ICE traffic needs to sent
type OutboundCallback func(raw []byte, local *stun.TransportAddr, remote *net.UDPAddr)

func newCandidatePair(local, remote Candidate) CandidatePair {
	return CandidatePair{
		remote: remote,
		local:  local,
	}
}

// CandidatePair represents a combination of a local and remote candidate
type CandidatePair struct {
	// lastUpdateTime ?
	remote Candidate
	local  Candidate
}

// GetAddrs returns network addresses for the candidate pair
func (c CandidatePair) GetAddrs() (local *stun.TransportAddr, remote *net.UDPAddr) {
	return &stun.TransportAddr{
			IP:   net.ParseIP(c.local.GetBase().Address),
			Port: c.local.GetBase().Port,
		}, &net.UDPAddr{
			IP:   net.ParseIP(c.remote.GetBase().Address),
			Port: c.remote.GetBase().Port,
		}
}

// Agent represents the ICE agent
type Agent struct {
	sync.RWMutex

	outboundCallback OutboundCallback
	iceNotifier      func(ConnectionState)

	tieBreaker      uint64
	connectionState ConnectionState
	gatheringState  GatheringState

	haveStarted   bool
	isControlling bool
	taskLoopChan  chan bool

	LocalUfrag      string
	LocalPwd        string
	LocalCandidates []Candidate

	remoteUfrag      string
	remotePwd        string
	remoteCandidates []Candidate

	selectedPair CandidatePair
	validPairs   []CandidatePair
}

const (
	agentTickerBaseInterval = 3 * time.Second
	stunTimeout             = 10 * time.Second
)

// NewAgent creates a new Agent
func NewAgent(outboundCallback OutboundCallback, iceNotifier func(ConnectionState)) *Agent {
	return &Agent{
		outboundCallback: outboundCallback,
		iceNotifier:      iceNotifier,

		tieBreaker:      rand.Uint64(),
		gatheringState:  GatheringStateComplete, // TODO trickle-ice
		connectionState: ConnectionStateNew,

		LocalUfrag: util.RandSeq(16),
		LocalPwd:   util.RandSeq(32),
	}
}

// Start starts the agent
func (a *Agent) Start(isControlling bool, remoteUfrag, remotePwd string) error {
	a.Lock()
	defer a.Unlock()

	if a.haveStarted {
		return errors.Errorf("Attempted to start agent twice")
	} else if remoteUfrag == "" {
		return errors.Errorf("remoteUfrag is empty")
	} else if remotePwd == "" {
		return errors.Errorf("remotePwd is empty")
	}

	a.isControlling = isControlling
	a.remoteUfrag = remoteUfrag
	a.remotePwd = remotePwd

	go a.agentTaskLoop()
	return nil
}

func (a *Agent) pingCandidate(local, remote Candidate) {
	msg, err := stun.Build(stun.ClassRequest, stun.MethodBinding, stun.GenerateTransactionId(),
		&stun.Username{Username: a.remoteUfrag + ":" + a.LocalUfrag},
		&stun.UseCandidate{},
		&stun.IceControlling{TieBreaker: a.tieBreaker},
		&stun.Priority{Priority: uint32(local.GetBase().Priority(HostCandidatePreference, 1))},
		&stun.MessageIntegrity{
			Key: []byte(a.remotePwd),
		},
		&stun.Fingerprint{},
	)
	if err != nil {
		fmt.Println(err)
		return
	}

	a.outboundCallback(msg.Pack(), &stun.TransportAddr{
		IP:   net.ParseIP(local.GetBase().Address),
		Port: local.GetBase().Port,
	}, &net.UDPAddr{
		IP:   net.ParseIP(remote.GetBase().Address),
		Port: remote.GetBase().Port,
	})
}

func (a *Agent) updateConnectionState(newState ConnectionState) {
	a.connectionState = newState
	// Call handler async since we may be holding the agent lock
	// and the handler may also require it
	go a.iceNotifier(a.connectionState)
}

func (a *Agent) setValidPair(local, remote Candidate, selected bool) {
	p := newCandidatePair(local, remote)

	if selected {
		a.selectedPair = p
		a.validPairs = nil
		// TODO: only set state to connected on selecting final pair?
		a.updateConnectionState(ConnectionStateConnected)
	} else {
		// keep track of pairs with succesfull bindings since any of them
		// can be used for communication until the final pair is selected:
		// https://tools.ietf.org/html/draft-ietf-ice-rfc5245bis-20#section-12
		a.validPairs = append(a.validPairs, p)
	}
}

func (a *Agent) agentTaskLoop() {
	// TODO this should be dynamic, and grow when the connection is stable
	t := time.NewTicker(agentTickerBaseInterval)
	a.updateConnectionState(ConnectionStateChecking)

	assertSelectedPairValid := func() bool {
		if a.selectedPair.remote == nil || a.selectedPair.local == nil {
			return false
		} else if time.Since(a.selectedPair.remote.GetBase().LastSeen) > stunTimeout {
			a.selectedPair.remote = nil
			a.selectedPair.local = nil
			a.updateConnectionState(ConnectionStateDisconnected)
			return false
		}

		return true
	}

	for {
		select {
		case <-t.C:
			a.Lock()
			if a.isControlling {
				if assertSelectedPairValid() {
					a.Unlock()
					continue
				}

				for _, localCandidate := range a.LocalCandidates {
					for _, remoteCandidate := range a.remoteCandidates {
						a.pingCandidate(localCandidate, remoteCandidate)
					}
				}
			} else {
				assertSelectedPairValid()
			}
			a.Unlock()
		case <-a.taskLoopChan:
			t.Stop()
			return
		}
	}
}

// AddRemoteCandidate adds a new remote candidate
func (a *Agent) AddRemoteCandidate(c Candidate) {
	a.Lock()
	defer a.Unlock()
	a.remoteCandidates = append(a.remoteCandidates, c)
}

// AddLocalCandidate adds a new local candidate
func (a *Agent) AddLocalCandidate(c Candidate) {
	a.Lock()
	defer a.Unlock()
	a.LocalCandidates = append(a.LocalCandidates, c)
}

// Close cleans up the Agent
func (a *Agent) Close() {
	close(a.taskLoopChan)
}

func isCandidateMatch(c Candidate, testAddress string, testPort int) bool {
	if c.GetBase().Address == testAddress && c.GetBase().Port == testPort {
		return true
	}

	switch c := c.(type) {
	case *CandidateSrflx:
		if c.RemoteAddress == testAddress && c.RemotePort == testPort {
			return true
		}
	}

	return false
}

func getTransportAddrCandidate(candidates []Candidate, addr *stun.TransportAddr) Candidate {
	for _, c := range candidates {
		if isCandidateMatch(c, addr.IP.String(), addr.Port) {
			return c
		}
	}
	return nil
}

func getUDPAddrCandidate(candidates []Candidate, addr *net.UDPAddr) Candidate {
	for _, c := range candidates {
		if isCandidateMatch(c, addr.IP.String(), addr.Port) {
			return c
		}
	}
	return nil
}

func (a *Agent) sendBindingSuccess(m *stun.Message, local *stun.TransportAddr, remote *net.UDPAddr) {
	if out, err := stun.Build(stun.ClassSuccessResponse, stun.MethodBinding, m.TransactionID,
		&stun.XorMappedAddress{
			XorAddress: stun.XorAddress{
				IP:   remote.IP,
				Port: remote.Port,
			},
		},
		&stun.MessageIntegrity{
			Key: []byte(a.LocalPwd),
		},
		&stun.Fingerprint{},
	); err != nil {
		fmt.Printf("Failed to handle inbound ICE from: %s to: %s error: %s", local.String(), remote.String(), err.Error())
	} else {
		a.outboundCallback(out.Pack(), local, remote)
	}

}

func (a *Agent) handleInboundControlled(m *stun.Message, local *stun.TransportAddr, remote *net.UDPAddr, localCandidate, remoteCandidate Candidate) {
	if _, isControlled := m.GetOneAttribute(stun.AttrIceControlled); isControlled && !a.isControlling {
		fmt.Println("inbound isControlled && a.isControlling == false")
		return
	}

	_, useCandidateFound := m.GetOneAttribute(stun.AttrUseCandidate)
	a.setValidPair(localCandidate, remoteCandidate, useCandidateFound)

	a.sendBindingSuccess(m, local, remote)
}

func (a *Agent) handleInboundControlling(m *stun.Message, local *stun.TransportAddr, remote *net.UDPAddr, localCandidate, remoteCandidate Candidate) {
	if _, isControlling := m.GetOneAttribute(stun.AttrIceControlling); isControlling && a.isControlling {
		fmt.Println("inbound isControlling && a.isControlling == true")
		return
	} else if _, useCandidate := m.GetOneAttribute(stun.AttrUseCandidate); useCandidate && a.isControlling {
		fmt.Println("useCandidate && a.isControlling == true")
		return
	}

	final := m.Class == stun.ClassSuccessResponse && m.Method == stun.MethodBinding
	a.setValidPair(localCandidate, remoteCandidate, final)

	if !final {
		a.sendBindingSuccess(m, local, remote)
	}
}

// HandleInbound processes traffic from a remote candidate
func (a *Agent) HandleInbound(buf []byte, local *stun.TransportAddr, remote *net.UDPAddr) {
	a.Lock()
	defer a.Unlock()

	localCandidate := getTransportAddrCandidate(a.LocalCandidates, local)
	if localCandidate == nil {
		// TODO debug
		// fmt.Printf("Could not find local candidate for %s:%d ", local.IP.String(), local.Value)
		return
	}

	remoteCandidate := getUDPAddrCandidate(a.remoteCandidates, remote)
	if remoteCandidate == nil {
		// TODO debug
		// fmt.Printf("Could not find remote candidate for %s:%d ", remote.IP.String(), remote.Value)
		return
	}
	remoteCandidate.GetBase().LastSeen = time.Now()

	m, err := stun.NewMessage(buf)
	if err != nil {
		fmt.Println(fmt.Sprintf("Failed to handle decode ICE from: %s to: %s error: %s", local.String(), remote.String(), err.Error()))
		return
	}

	if a.isControlling {
		a.handleInboundControlling(m, local, remote, localCandidate, remoteCandidate)
	} else {
		a.handleInboundControlled(m, local, remote, localCandidate, remoteCandidate)
	}

}

// SelectedPair gets the current selected pair's Addresses (or returns nil)
func (a *Agent) SelectedPair() (local *stun.TransportAddr, remote *net.UDPAddr) {
	a.RLock()
	defer a.RUnlock()

	if a.selectedPair.remote == nil || a.selectedPair.local == nil {
		for _, p := range a.validPairs {
			return p.GetAddrs()
		}
		return nil, nil
	}

	return a.selectedPair.GetAddrs()
}
<commit_msg>Fix a unittest failure due to closing nil channel<commit_after>package ice

import (
	"fmt"
	"math/rand"
	"net"
	"sync"
	"time"

	"github.com/pions/pkg/stun"
	"github.com/pions/webrtc/internal/util"
	"github.com/pkg/errors"
)

const Unknown = iota

// OutboundCallback is the user defined Callback that is called when ICE traffic needs to sent
type OutboundCallback func(raw []byte, local *stun.TransportAddr, remote *net.UDPAddr)

func newCandidatePair(local, remote Candidate) CandidatePair {
	return CandidatePair{
		remote: remote,
		local:  local,
	}
}

// CandidatePair represents a combination of a local and remote candidate
type CandidatePair struct {
	// lastUpdateTime ?
	remote Candidate
	local  Candidate
}

// GetAddrs returns network addresses for the candidate pair
func (c CandidatePair) GetAddrs() (local *stun.TransportAddr, remote *net.UDPAddr) {
	return &stun.TransportAddr{
			IP:   net.ParseIP(c.local.GetBase().Address),
			Port: c.local.GetBase().Port,
		}, &net.UDPAddr{
			IP:   net.ParseIP(c.remote.GetBase().Address),
			Port: c.remote.GetBase().Port,
		}
}

// Agent represents the ICE agent
type Agent struct {
	sync.RWMutex

	outboundCallback OutboundCallback
	iceNotifier      func(ConnectionState)

	tieBreaker      uint64
	connectionState ConnectionState
	gatheringState  GatheringState

	haveStarted   bool
	isControlling bool
	taskLoopChan  chan bool

	LocalUfrag      string
	LocalPwd        string
	LocalCandidates []Candidate

	remoteUfrag      string
	remotePwd        string
	remoteCandidates []Candidate

	selectedPair CandidatePair
	validPairs   []CandidatePair
}

const (
	agentTickerBaseInterval = 3 * time.Second
	stunTimeout             = 10 * time.Second
)

// NewAgent creates a new Agent
func NewAgent(outboundCallback OutboundCallback, iceNotifier func(ConnectionState)) *Agent {
	return &Agent{
		outboundCallback: outboundCallback,
		iceNotifier:      iceNotifier,

		tieBreaker:      rand.Uint64(),
		gatheringState:  GatheringStateComplete, // TODO trickle-ice
		connectionState: ConnectionStateNew,

		LocalUfrag: util.RandSeq(16),
		LocalPwd:   util.RandSeq(32),
	}
}

// Start starts the agent
func (a *Agent) Start(isControlling bool, remoteUfrag, remotePwd string) error {
	a.Lock()
	defer a.Unlock()

	if a.haveStarted {
		return errors.Errorf("Attempted to start agent twice")
	} else if remoteUfrag == "" {
		return errors.Errorf("remoteUfrag is empty")
	} else if remotePwd == "" {
		return errors.Errorf("remotePwd is empty")
	}

	a.isControlling = isControlling
	a.remoteUfrag = remoteUfrag
	a.remotePwd = remotePwd

	go a.agentTaskLoop()
	return nil
}

func (a *Agent) pingCandidate(local, remote Candidate) {
	msg, err := stun.Build(stun.ClassRequest, stun.MethodBinding, stun.GenerateTransactionId(),
		&stun.Username{Username: a.remoteUfrag + ":" + a.LocalUfrag},
		&stun.UseCandidate{},
		&stun.IceControlling{TieBreaker: a.tieBreaker},
		&stun.Priority{Priority: uint32(local.GetBase().Priority(HostCandidatePreference, 1))},
		&stun.MessageIntegrity{
			Key: []byte(a.remotePwd),
		},
		&stun.Fingerprint{},
	)
	if err != nil {
		fmt.Println(err)
		return
	}

	a.outboundCallback(msg.Pack(), &stun.TransportAddr{
		IP:   net.ParseIP(local.GetBase().Address),
		Port: local.GetBase().Port,
	}, &net.UDPAddr{
		IP:   net.ParseIP(remote.GetBase().Address),
		Port: remote.GetBase().Port,
	})
}

func (a *Agent) updateConnectionState(newState ConnectionState) {
	a.connectionState = newState
	// Call handler async since we may be holding the agent lock
	// and the handler may also require it
	go a.iceNotifier(a.connectionState)
}

func (a *Agent) setValidPair(local, remote Candidate, selected bool) {
	p := newCandidatePair(local, remote)

	if selected {
		a.selectedPair = p
		a.validPairs = nil
		// TODO: only set state to connected on selecting final pair?
		a.updateConnectionState(ConnectionStateConnected)
	} else {
		// keep track of pairs with succesfull bindings since any of them
		// can be used for communication until the final pair is selected:
		// https://tools.ietf.org/html/draft-ietf-ice-rfc5245bis-20#section-12
		a.validPairs = append(a.validPairs, p)
	}
}

func (a *Agent) agentTaskLoop() {
	// TODO this should be dynamic, and grow when the connection is stable
	t := time.NewTicker(agentTickerBaseInterval)
	a.updateConnectionState(ConnectionStateChecking)

	assertSelectedPairValid := func() bool {
		if a.selectedPair.remote == nil || a.selectedPair.local == nil {
			return false
		} else if time.Since(a.selectedPair.remote.GetBase().LastSeen) > stunTimeout {
			a.selectedPair.remote = nil
			a.selectedPair.local = nil
			a.updateConnectionState(ConnectionStateDisconnected)
			return false
		}

		return true
	}

	for {
		select {
		case <-t.C:
			a.Lock()
			if a.isControlling {
				if assertSelectedPairValid() {
					a.Unlock()
					continue
				}

				for _, localCandidate := range a.LocalCandidates {
					for _, remoteCandidate := range a.remoteCandidates {
						a.pingCandidate(localCandidate, remoteCandidate)
					}
				}
			} else {
				assertSelectedPairValid()
			}
			a.Unlock()
		case <-a.taskLoopChan:
			t.Stop()
			return
		}
	}
}

// AddRemoteCandidate adds a new remote candidate
func (a *Agent) AddRemoteCandidate(c Candidate) {
	a.Lock()
	defer a.Unlock()
	a.remoteCandidates = append(a.remoteCandidates, c)
}

// AddLocalCandidate adds a new local candidate
func (a *Agent) AddLocalCandidate(c Candidate) {
	a.Lock()
	defer a.Unlock()
	a.LocalCandidates = append(a.LocalCandidates, c)
}

// Close cleans up the Agent
func (a *Agent) Close() {
	if a.taskLoopChan != nil {
		close(a.taskLoopChan)
	}
}

func isCandidateMatch(c Candidate, testAddress string, testPort int) bool {
	if c.GetBase().Address == testAddress && c.GetBase().Port == testPort {
		return true
	}

	switch c := c.(type) {
	case *CandidateSrflx:
		if c.RemoteAddress == testAddress && c.RemotePort == testPort {
			return true
		}
	}

	return false
}

func getTransportAddrCandidate(candidates []Candidate, addr *stun.TransportAddr) Candidate {
	for _, c := range candidates {
		if isCandidateMatch(c, addr.IP.String(), addr.Port) {
			return c
		}
	}
	return nil
}

func getUDPAddrCandidate(candidates []Candidate, addr *net.UDPAddr) Candidate {
	for _, c := range candidates {
		if isCandidateMatch(c, addr.IP.String(), addr.Port) {
			return c
		}
	}
	return nil
}

func (a *Agent) sendBindingSuccess(m *stun.Message, local *stun.TransportAddr, remote *net.UDPAddr) {
	if out, err := stun.Build(stun.ClassSuccessResponse, stun.MethodBinding, m.TransactionID,
		&stun.XorMappedAddress{
			XorAddress: stun.XorAddress{
				IP:   remote.IP,
				Port: remote.Port,
			},
		},
		&stun.MessageIntegrity{
			Key: []byte(a.LocalPwd),
		},
		&stun.Fingerprint{},
	); err != nil {
		fmt.Printf("Failed to handle inbound ICE from: %s to: %s error: %s", local.String(), remote.String(), err.Error())
	} else {
		a.outboundCallback(out.Pack(), local, remote)
	}

}

func (a *Agent) handleInboundControlled(m *stun.Message, local *stun.TransportAddr, remote *net.UDPAddr, localCandidate, remoteCandidate Candidate) {
	if _, isControlled := m.GetOneAttribute(stun.AttrIceControlled); isControlled && !a.isControlling {
		fmt.Println("inbound isControlled && a.isControlling == false")
		return
	}

	_, useCandidateFound := m.GetOneAttribute(stun.AttrUseCandidate)
	a.setValidPair(localCandidate, remoteCandidate, useCandidateFound)

	a.sendBindingSuccess(m, local, remote)
}

func (a *Agent) handleInboundControlling(m *stun.Message, local *stun.TransportAddr, remote *net.UDPAddr, localCandidate, remoteCandidate Candidate) {
	if _, isControlling := m.GetOneAttribute(stun.AttrIceControlling); isControlling && a.isControlling {
		fmt.Println("inbound isControlling && a.isControlling == true")
		return
	} else if _, useCandidate := m.GetOneAttribute(stun.AttrUseCandidate); useCandidate && a.isControlling {
		fmt.Println("useCandidate && a.isControlling == true")
		return
	}

	final := m.Class == stun.ClassSuccessResponse && m.Method == stun.MethodBinding
	a.setValidPair(localCandidate, remoteCandidate, final)

	if !final {
		a.sendBindingSuccess(m, local, remote)
	}
}

// HandleInbound processes traffic from a remote candidate
func (a *Agent) HandleInbound(buf []byte, local *stun.TransportAddr, remote *net.UDPAddr) {
	a.Lock()
	defer a.Unlock()

	localCandidate := getTransportAddrCandidate(a.LocalCandidates, local)
	if localCandidate == nil {
		// TODO debug
		// fmt.Printf("Could not find local candidate for %s:%d ", local.IP.String(), local.Value)
		return
	}

	remoteCandidate := getUDPAddrCandidate(a.remoteCandidates, remote)
	if remoteCandidate == nil {
		// TODO debug
		// fmt.Printf("Could not find remote candidate for %s:%d ", remote.IP.String(), remote.Value)
		return
	}
	remoteCandidate.GetBase().LastSeen = time.Now()

	m, err := stun.NewMessage(buf)
	if err != nil {
		fmt.Println(fmt.Sprintf("Failed to handle decode ICE from: %s to: %s error: %s", local.String(), remote.String(), err.Error()))
		return
	}

	if a.isControlling {
		a.handleInboundControlling(m, local, remote, localCandidate, remoteCandidate)
	} else {
		a.handleInboundControlled(m, local, remote, localCandidate, remoteCandidate)
	}

}

// SelectedPair gets the current selected pair's Addresses (or returns nil)
func (a *Agent) SelectedPair() (local *stun.TransportAddr, remote *net.UDPAddr) {
	a.RLock()
	defer a.RUnlock()

	if a.selectedPair.remote == nil || a.selectedPair.local == nil {
		for _, p := range a.validPairs {
			return p.GetAddrs()
		}
		return nil, nil
	}

	return a.selectedPair.GetAddrs()
}
<|endoftext|>