<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws

import (
	"errors"
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/golang/glog"
)

// autoScaling is the interface represents a specific aspect of the auto-scaling service provided by AWS SDK for use in CA
type autoScaling interface {
	DescribeAutoScalingGroups(input *autoscaling.DescribeAutoScalingGroupsInput) (*autoscaling.DescribeAutoScalingGroupsOutput, error)
	DescribeLaunchConfigurations(*autoscaling.DescribeLaunchConfigurationsInput) (*autoscaling.DescribeLaunchConfigurationsOutput, error)
	DescribeTags(input *autoscaling.DescribeTagsInput) (*autoscaling.DescribeTagsOutput, error)
	SetDesiredCapacity(input *autoscaling.SetDesiredCapacityInput) (*autoscaling.SetDesiredCapacityOutput, error)
	TerminateInstanceInAutoScalingGroup(input *autoscaling.TerminateInstanceInAutoScalingGroupInput) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error)
}

// autoScalingWrapper provides several utility methods over the auto-scaling service provided by AWS SDK
type autoScalingWrapper struct {
	autoScaling
}

func (m autoScalingWrapper) getInstanceTypeByLCName(name string) (string, error) {
	params := &autoscaling.DescribeLaunchConfigurationsInput{
		LaunchConfigurationNames: []*string{aws.String(name)},
		MaxRecords:               aws.Int64(1),
	}
	launchConfigurations, err := m.DescribeLaunchConfigurations(params)
	if err != nil {
		glog.V(4).Infof("Failed LaunchConfiguration info request for %s: %v", name, err)
		return "", err
	}
	if len(launchConfigurations.LaunchConfigurations) < 1 {
		return "", fmt.Errorf("Unable to get first LaunchConfiguration for %s", name)
	}

	return *launchConfigurations.LaunchConfigurations[0].InstanceType, nil
}

func (m autoScalingWrapper) getAutoscalingGroupByName(name string) (*autoscaling.Group, error) {
	params := &autoscaling.DescribeAutoScalingGroupsInput{
		AutoScalingGroupNames: []*string{aws.String(name)},
		MaxRecords:            aws.Int64(1),
	}
	groups, err := m.DescribeAutoScalingGroups(params)
	if err != nil {
		glog.V(4).Infof("Failed ASG info request for %s: %v", name, err)
		return nil, err
	}
	if len(groups.AutoScalingGroups) < 1 {
		return nil, fmt.Errorf("Unable to get first autoscaling.Group for %s", name)
	}
	return groups.AutoScalingGroups[0], nil
}

func (m *autoScalingWrapper) getAutoscalingGroupsByNames(names []string) ([]*autoscaling.Group, error) {
	glog.V(6).Infof("Starting getAutoscalingGroupsByNames with names=%v", names)

	nameRefs := []*string{}
	for _, n := range names {
		nameRefs = append(nameRefs, aws.String(n))
	}
	params := &autoscaling.DescribeAutoScalingGroupsInput{
		AutoScalingGroupNames: nameRefs,
		MaxRecords:            aws.Int64(maxRecordsReturnedByAPI),
	}
	description, err := m.DescribeAutoScalingGroups(params)
	if err != nil {
		glog.V(4).Infof("Failed to describe ASGs : %v", err)
		return nil, err
	}
	if len(description.AutoScalingGroups) < 1 {
		return nil, errors.New("No ASGs found")
	}

	asgs := description.AutoScalingGroups
	for description.NextToken != nil {
		description, err = m.DescribeAutoScalingGroups(&autoscaling.DescribeAutoScalingGroupsInput{
			NextToken:  description.NextToken,
			MaxRecords: aws.Int64(maxRecordsReturnedByAPI),
		})
		if err != nil {
			glog.V(4).Infof("Failed to describe ASGs : %v", err)
			return nil, err
		}
		asgs = append(asgs, description.AutoScalingGroups...)
	}

	glog.V(6).Infof("Finishing getAutoscalingGroupsByNames asgs=%v", asgs)

	return asgs, nil
}

func (m *autoScalingWrapper) getAutoscalingGroupsByTags(keys []string) ([]*autoscaling.Group, error) {
	glog.V(6).Infof("Starting getAutoscalingGroupsByTag with keys=%v", keys)

	numKeys := len(keys)

	// DescribeTags does an OR query when multiple filters on different tags are specified.
	// In other words, DescribeTags returns [asg1, asg1] for keys [t1, t2] when there's only one asg tagged both t1 and t2.
	filters := []*autoscaling.Filter{}
	for _, key := range keys {
		filter := &autoscaling.Filter{
			Name:   aws.String("key"),
			Values: []*string{aws.String(key)},
		}
		filters = append(filters, filter)
	}
	description, err := m.DescribeTags(&autoscaling.DescribeTagsInput{
		Filters:    filters,
		MaxRecords: aws.Int64(maxRecordsReturnedByAPI),
	})
	if err != nil {
		glog.V(4).Infof("Failed to describe ASG tags for keys %v : %v", keys, err)
		return nil, err
	}
	if len(description.Tags) < 1 {
		return nil, fmt.Errorf("Unable to find ASGs for tag keys %v", keys)
	}
	tags := []*autoscaling.TagDescription{}
	tags = append(tags, description.Tags...)

	for description.NextToken != nil {
		description, err = m.DescribeTags(&autoscaling.DescribeTagsInput{
			NextToken:  description.NextToken,
			MaxRecords: aws.Int64(maxRecordsReturnedByAPI),
		})
		if err != nil {
			glog.V(4).Infof("Failed to describe ASG tags for key %v: %v", keys, err)
			return nil, err
		}
		tags = append(tags, description.Tags...)
	}

	// De-duplicate asg names
	asgNameOccurrences := map[string]int{}
	for _, t := range tags {
		asgName := *(t.ResourceId)
		if n, ok := asgNameOccurrences[asgName]; ok {
			asgNameOccurrences[asgName] = n + 1
		} else {
			asgNameOccurrences[asgName] = 1
		}
	}
	// Accordingly to how DescribeTags API works, the result contains ASGs which not all but only subset of tags are associated.
	// Explicitly select ASGs to which all the tags are associated so that we won't end up calling DescribeAutoScalingGroups API
	// multiple times on an ASG
	asgNames := []string{}
	for asgName, n := range asgNameOccurrences {
		if n == numKeys {
			asgNames = append(asgNames, asgName)
		}
	}

	asgs, err := m.getAutoscalingGroupsByNames(asgNames)
	if err != nil {
		return nil, err
	}

	glog.V(6).Infof("Finishing getAutoscalingGroupsByTag with asgs=%v", asgs)

	return asgs, nil
}
<commit_msg>Check ASG name list not empty<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws

import (
	"errors"
	"fmt"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/autoscaling"
	"github.com/golang/glog"
)

// autoScaling is the interface represents a specific aspect of the auto-scaling service provided by AWS SDK for use in CA
type autoScaling interface {
	DescribeAutoScalingGroups(input *autoscaling.DescribeAutoScalingGroupsInput) (*autoscaling.DescribeAutoScalingGroupsOutput, error)
	DescribeLaunchConfigurations(*autoscaling.DescribeLaunchConfigurationsInput) (*autoscaling.DescribeLaunchConfigurationsOutput, error)
	DescribeTags(input *autoscaling.DescribeTagsInput) (*autoscaling.DescribeTagsOutput, error)
	SetDesiredCapacity(input *autoscaling.SetDesiredCapacityInput) (*autoscaling.SetDesiredCapacityOutput, error)
	TerminateInstanceInAutoScalingGroup(input *autoscaling.TerminateInstanceInAutoScalingGroupInput) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error)
}

// autoScalingWrapper provides several utility methods over the auto-scaling service provided by AWS SDK
type autoScalingWrapper struct {
	autoScaling
}

func (m autoScalingWrapper) getInstanceTypeByLCName(name string) (string, error) {
	params := &autoscaling.DescribeLaunchConfigurationsInput{
		LaunchConfigurationNames: []*string{aws.String(name)},
		MaxRecords:               aws.Int64(1),
	}
	launchConfigurations, err := m.DescribeLaunchConfigurations(params)
	if err != nil {
		glog.V(4).Infof("Failed LaunchConfiguration info request for %s: %v", name, err)
		return "", err
	}
	if len(launchConfigurations.LaunchConfigurations) < 1 {
		return "", fmt.Errorf("Unable to get first LaunchConfiguration for %s", name)
	}

	return *launchConfigurations.LaunchConfigurations[0].InstanceType, nil
}

func (m autoScalingWrapper) getAutoscalingGroupByName(name string) (*autoscaling.Group, error) {
	params := &autoscaling.DescribeAutoScalingGroupsInput{
		AutoScalingGroupNames: []*string{aws.String(name)},
		MaxRecords:            aws.Int64(1),
	}
	groups, err := m.DescribeAutoScalingGroups(params)
	if err != nil {
		glog.V(4).Infof("Failed ASG info request for %s: %v", name, err)
		return nil, err
	}
	if len(groups.AutoScalingGroups) < 1 {
		return nil, fmt.Errorf("Unable to get first autoscaling.Group for %s", name)
	}
	return groups.AutoScalingGroups[0], nil
}

func (m *autoScalingWrapper) getAutoscalingGroupsByNames(names []string) ([]*autoscaling.Group, error) {
	glog.V(6).Infof("Starting getAutoscalingGroupsByNames with names=%v", names)

	if len(names) < 1 {
		glog.V(4).Info("Failed to describe ASGs: Must specify at least one ASG name.")
		return nil, fmt.Errorf("List of ASG names was empty. Must specify at least one ASG name")
	}

	nameRefs := []*string{}
	for _, n := range names {
		nameRefs = append(nameRefs, aws.String(n))
	}
	params := &autoscaling.DescribeAutoScalingGroupsInput{
		AutoScalingGroupNames: nameRefs,
		MaxRecords:            aws.Int64(maxRecordsReturnedByAPI),
	}
	description, err := m.DescribeAutoScalingGroups(params)
	if err != nil {
		glog.V(4).Infof("Failed to describe ASGs : %v", err)
		return nil, err
	}
	if len(description.AutoScalingGroups) < 1 {
		return nil, errors.New("No ASGs found")
	}

	asgs := description.AutoScalingGroups
	for description.NextToken != nil {
		description, err = m.DescribeAutoScalingGroups(&autoscaling.DescribeAutoScalingGroupsInput{
			NextToken:  description.NextToken,
			MaxRecords: aws.Int64(maxRecordsReturnedByAPI),
		})
		if err != nil {
			glog.V(4).Infof("Failed to describe ASGs : %v", err)
			return nil, err
		}
		asgs = append(asgs, description.AutoScalingGroups...)
	}

	glog.V(6).Infof("Finishing getAutoscalingGroupsByNames asgs=%v", asgs)

	return asgs, nil
}

func (m *autoScalingWrapper) getAutoscalingGroupsByTags(keys []string) ([]*autoscaling.Group, error) {
	glog.V(6).Infof("Starting getAutoscalingGroupsByTag with keys=%v", keys)

	numKeys := len(keys)

	// DescribeTags does an OR query when multiple filters on different tags are specified.
	// In other words, DescribeTags returns [asg1, asg1] for keys [t1, t2] when there's only one asg tagged both t1 and t2.
	filters := []*autoscaling.Filter{}
	for _, key := range keys {
		filter := &autoscaling.Filter{
			Name:   aws.String("key"),
			Values: []*string{aws.String(key)},
		}
		filters = append(filters, filter)
	}
	description, err := m.DescribeTags(&autoscaling.DescribeTagsInput{
		Filters:    filters,
		MaxRecords: aws.Int64(maxRecordsReturnedByAPI),
	})
	if err != nil {
		glog.V(4).Infof("Failed to describe ASG tags for keys %v : %v", keys, err)
		return nil, err
	}
	if len(description.Tags) < 1 {
		return nil, fmt.Errorf("Unable to find ASGs for tag keys %v", keys)
	}
	tags := []*autoscaling.TagDescription{}
	tags = append(tags, description.Tags...)

	for description.NextToken != nil {
		description, err = m.DescribeTags(&autoscaling.DescribeTagsInput{
			NextToken:  description.NextToken,
			MaxRecords: aws.Int64(maxRecordsReturnedByAPI),
		})
		if err != nil {
			glog.V(4).Infof("Failed to describe ASG tags for key %v: %v", keys, err)
			return nil, err
		}
		tags = append(tags, description.Tags...)
	}

	// De-duplicate asg names
	asgNameOccurrences := map[string]int{}
	for _, t := range tags {
		asgName := *(t.ResourceId)
		if n, ok := asgNameOccurrences[asgName]; ok {
			asgNameOccurrences[asgName] = n + 1
		} else {
			asgNameOccurrences[asgName] = 1
		}
	}
	// Accordingly to how DescribeTags API works, the result contains ASGs which not all but only subset of tags are associated.
	// Explicitly select ASGs to which all the tags are associated so that we won't end up calling DescribeAutoScalingGroups API
	// multiple times on an ASG
	asgNames := []string{}
	for asgName, n := range asgNameOccurrences {
		if n == numKeys {
			asgNames = append(asgNames, asgName)
		}
	}

	asgs, err := m.getAutoscalingGroupsByNames(asgNames)
	if err != nil {
		return nil, err
	}

	glog.V(6).Infof("Finishing getAutoscalingGroupsByTag with asgs=%v", asgs)

	return asgs, nil
}
<|endoftext|>


<commit_before>package drive

import (
    "fmt"
    "io"
    "os"
    "time"
    "sort"
    "bytes"
    "path/filepath"
    "google.golang.org/api/googleapi"
    "google.golang.org/api/drive/v3"
)

type UploadSyncArgs struct {
    Out io.Writer
    Progress io.Writer
    Path string
    RootId string
    DryRun bool
    DeleteExtraneous bool
    ChunkSize int64
    Resolution ConflictResolution
    Comparer FileComparer
}

func (self *Drive) UploadSync(args UploadSyncArgs) error {
    if args.ChunkSize > intMax() - 1 {
        return fmt.Errorf("Chunk size is to big, max chunk size for this computer is %d", intMax() - 1)
    }

    fmt.Fprintln(args.Out, "Starting sync...")
    started := time.Now()

    // Create root directory if it does not exist
    rootDir, err := self.prepareSyncRoot(args)
    if err != nil {
        return err
    }

    fmt.Fprintln(args.Out, "Collecting local and remote file information...")
    files, err := self.prepareSyncFiles(args.Path, rootDir, args.Comparer)
    if err != nil {
        return err
    }

    // Find changed files
    changedFiles := files.filterChangedLocalFiles()

    fmt.Fprintf(args.Out, "Found %d local files and %d remote files\n", len(files.local), len(files.remote))

    // Ensure that that we don't overwrite any remote changes
    if args.Resolution == NoResolution {
        err = ensureNoRemoteModifications(changedFiles)
        if err != nil {
            return fmt.Errorf("Conflict detected!\nThe following files have changed and the remote file are newer than it's local counterpart:\n\n%s\nNo conflict resolution was given, aborting...", err)
        }
    }

    // Create missing directories
    files, err = self.createMissingRemoteDirs(files, args)
    if err != nil {
        return err
    }

    // Upload missing files
    err = self.uploadMissingFiles(files, args)
    if err != nil {
        return err
    }

    // Update modified files
    err = self.updateChangedFiles(changedFiles, rootDir, args)
    if err != nil {
        return err
    }

    // Delete extraneous files on drive
    if args.DeleteExtraneous {
        err = self.deleteExtraneousRemoteFiles(files, args)
        if err != nil {
            return err
        }
    }
    fmt.Fprintf(args.Out, "Sync finished in %s\n", time.Since(started))

    return nil
}

func (self *Drive) prepareSyncRoot(args UploadSyncArgs) (*drive.File, error) {
    fields := []googleapi.Field{"id", "name", "mimeType", "appProperties"}
    f, err := self.service.Files.Get(args.RootId).Fields(fields...).Do()
    if err != nil {
        return nil, fmt.Errorf("Failed to find root dir: %s", err)
    }

    // Ensure file is a directory
    if !isDir(f) {
        return nil, fmt.Errorf("Provided root id is not a directory")
    }

    // Return directory if syncRoot property is already set
    if _, ok := f.AppProperties["isSyncRoot"]; ok {
        return f, nil
    }

    // This is the first time this directory have been used for sync
    // Check if the directory is empty
    isEmpty, err := self.dirIsEmpty(f.Id)
    if err != nil {
        return nil, fmt.Errorf("Failed to check if root dir is empty: %s", err)
    }

    // Ensure that the directory is empty
    if !isEmpty {
        return nil, fmt.Errorf("Root directoy is not empty, the initial sync requires an empty directory")
    }

    // Update directory with syncRoot property
    dstFile := &drive.File{
        AppProperties: map[string]string{"isSyncRoot": "true"},
    }

    f, err = self.service.Files.Update(f.Id, dstFile).Fields(fields...).Do()
    if err != nil {
        return nil, fmt.Errorf("Failed to update root directory: %s", err)
    }

    return f, nil
}

func (self *Drive) createMissingRemoteDirs(files *syncFiles, args UploadSyncArgs) (*syncFiles, error) {
    missingDirs := files.filterMissingRemoteDirs()
    missingCount := len(missingDirs)

    if missingCount > 0 {
        fmt.Fprintf(args.Out, "\n%d remote directories are missing\n", missingCount)
    }

    // Sort directories so that the dirs with the shortest path comes first
    sort.Sort(byLocalPathLength(missingDirs))

    for i, lf := range missingDirs {
        parentPath := parentFilePath(lf.relPath)
        parent, ok := files.findRemoteByPath(parentPath)
        if !ok {
            return nil, fmt.Errorf("Could not find remote directory with path '%s'", parentPath)
        }

        fmt.Fprintf(args.Out, "[%04d/%04d] Creating directory %s\n", i + 1, missingCount, filepath.Join(files.root.file.Name, lf.relPath))

        f, err := self.createMissingRemoteDir(createMissingRemoteDirArgs{
            name: lf.info.Name(),
            parentId: parent.file.Id,
            rootId: args.RootId,
            dryRun: args.DryRun,
            try: 0,
        })
        if err != nil {
            return nil, err
        }

        files.remote = append(files.remote, &RemoteFile{
            relPath: lf.relPath,
            file: f,
        })
    }

    return files, nil
}

type createMissingRemoteDirArgs struct {
    name string
    parentId string
    rootId string
    dryRun bool
    try int
}

func (self *Drive) uploadMissingFiles(files *syncFiles, args UploadSyncArgs) error {
    missingFiles := files.filterMissingRemoteFiles()
    missingCount := len(missingFiles)

    if missingCount > 0 {
        fmt.Fprintf(args.Out, "\n%d remote files are missing\n", missingCount)
    }

    for i, lf := range missingFiles {
        parentPath := parentFilePath(lf.relPath)
        parent, ok := files.findRemoteByPath(parentPath)
        if !ok {
            return fmt.Errorf("Could not find remote directory with path '%s'", parentPath)
        }

        fmt.Fprintf(args.Out, "[%04d/%04d] Uploading %s -> %s\n", i + 1, missingCount, lf.relPath, filepath.Join(files.root.file.Name, lf.relPath))

        if args.DryRun {
            continue
        }

        err := self.uploadMissingFile(parent.file.Id, lf, args, 0)
        if err != nil {
            return err
        }
    }

    return nil
}

func (self *Drive) updateChangedFiles(changedFiles []*changedFile, root *drive.File, args UploadSyncArgs) error {
    changedCount := len(changedFiles)

    if changedCount > 0 {
        fmt.Fprintf(args.Out, "\n%d local files has changed\n", changedCount)
    }

    for i, cf := range changedFiles {
        if skip, reason := checkRemoteConflict(cf, args.Resolution); skip {
            fmt.Fprintf(args.Out, "[%04d/%04d] Skipping %s (%s)\n", i + 1, changedCount, cf.local.relPath, reason)
            continue
        }

        fmt.Fprintf(args.Out, "[%04d/%04d] Updating %s -> %s\n", i + 1, changedCount, cf.local.relPath, filepath.Join(root.Name, cf.local.relPath))

        if args.DryRun {
            continue
        }

        err := self.updateChangedFile(cf, args, 0)
        if err != nil {
            return err
        }
    }

    return nil
}

func (self *Drive) deleteExtraneousRemoteFiles(files *syncFiles, args UploadSyncArgs) error {
    extraneousFiles := files.filterExtraneousRemoteFiles()
    extraneousCount := len(extraneousFiles)

    if extraneousCount > 0 {
        fmt.Fprintf(args.Out, "\n%d remote files are extraneous\n", extraneousCount)
    }

    // Sort files so that the files with the longest path comes first
    sort.Sort(sort.Reverse(byRemotePathLength(extraneousFiles)))

    for i, rf := range extraneousFiles {
        fmt.Fprintf(args.Out, "[%04d/%04d] Deleting %s\n", i + 1, extraneousCount, filepath.Join(files.root.file.Name, rf.relPath))

        if args.DryRun {
            continue
        }

        err := self.deleteRemoteFile(rf, args, 0)
        if err != nil {
            return err
        }
    }

    return nil
}

func (self *Drive) createMissingRemoteDir(args createMissingRemoteDirArgs) (*drive.File, error) {
    dstFile := &drive.File{
        Name: args.name,
        MimeType: DirectoryMimeType,
        Parents: []string{args.parentId},
        AppProperties: map[string]string{"syncRootId": args.rootId},
    }

    if args.dryRun {
        return dstFile, nil
    }

    f, err := self.service.Files.Create(dstFile).Do()
    if err != nil {
        if isBackendError(err) && args.try < MaxBackendErrorRetries {
            exponentialBackoffSleep(args.try)
            args.try++
            self.createMissingRemoteDir(args)
        } else {
            return nil, fmt.Errorf("Failed to create directory: %s", err)
        }
    }

    return f, nil
}

func (self *Drive) uploadMissingFile(parentId string, lf *LocalFile, args UploadSyncArgs, try int) error {
    srcFile, err := os.Open(lf.absPath)
    if err != nil {
        return fmt.Errorf("Failed to open file: %s", err)
    }

    // Close file on function exit
    defer srcFile.Close()

    // Instantiate drive file
    dstFile := &drive.File{
        Name: lf.info.Name(),
        Parents: []string{parentId},
        AppProperties: map[string]string{"syncRootId": args.RootId},
    }

    // Chunk size option
    chunkSize := googleapi.ChunkSize(int(args.ChunkSize))

    // Wrap file in progress reader
    srcReader := getProgressReader(srcFile, args.Progress, lf.info.Size())

    _, err = self.service.Files.Create(dstFile).Fields("id", "name", "size", "md5Checksum").Media(srcReader, chunkSize).Do()
    if err != nil {
        if isBackendError(err) && try < MaxBackendErrorRetries {
            exponentialBackoffSleep(try)
            try++
            self.uploadMissingFile(parentId, lf, args, try)
        } else {
            return fmt.Errorf("Failed to upload file: %s", err)
        }
    }

    return nil
}

func (self *Drive) updateChangedFile(cf *changedFile, args UploadSyncArgs, try int) error {
    srcFile, err := os.Open(cf.local.absPath)
    if err != nil {
        return fmt.Errorf("Failed to open file: %s", err)
    }

    // Close file on function exit
    defer srcFile.Close()

    // Instantiate drive file
    dstFile := &drive.File{}

    // Chunk size option
    chunkSize := googleapi.ChunkSize(int(args.ChunkSize))

    // Wrap file in progress reader
    srcReader := getProgressReader(srcFile, args.Progress, cf.local.info.Size())

    _, err = self.service.Files.Update(cf.remote.file.Id, dstFile).Media(srcReader, chunkSize).Do()
    if err != nil {
        if isBackendError(err) && try < MaxBackendErrorRetries {
            exponentialBackoffSleep(try)
            try++
            self.updateChangedFile(cf, args, try)
        } else {
            return fmt.Errorf("Failed to update file: %s", err)
        }
    }

    return nil
}

func (self *Drive) deleteRemoteFile(rf *RemoteFile, args UploadSyncArgs, try int) error {
    err := self.service.Files.Delete(rf.file.Id).Do()
    if err != nil {
        if isBackendError(err) && try < MaxBackendErrorRetries {
            exponentialBackoffSleep(try)
            try++
            self.deleteRemoteFile(rf, args, try)
        } else {
            return fmt.Errorf("Failed to delete file: %s", err)
        }
    }

    return nil
}

func (self *Drive) dirIsEmpty(id string) (bool, error) {
    query := fmt.Sprintf("'%s' in parents", id)
    fileList, err := self.service.Files.List().Q(query).Do()
    if err != nil {
        return false, fmt.Errorf("Empty dir check failed: ", err)
    }

    return len(fileList.Files) == 0, nil
}

func checkRemoteConflict(cf *changedFile, resolution ConflictResolution) (bool, string) {
    // No conflict unless remote file was last modified
    if cf.compareModTime() != RemoteLastModified {
        return false, ""
    }

    // Don't skip if want to keep the local file
    if resolution == KeepLocal {
        return false, ""
    }

    // Skip if we want to keep the remote file
    if resolution == KeepRemote {
        return true, "conflicting file, keeping remote file"
    }

    if resolution == KeepLargest {
        largest := cf.compareSize()

        // Skip if the remote file is largest
        if largest == RemoteLargestSize {
            return true, "conflicting file, remote file is largest, keeping remote"
        }

        // Don't skip if the local file is largest
        if largest == LocalLargestSize {
            return false, ""
        }

        // Keep remote if both files have the same size
        if largest == EqualSize {
            return true, "conflicting file, file sizes are equal, keeping remote"
        }
    }

    // The conditionals above should cover all cases,
    // unless the programmer did something wrong,
    // in which case we default to being non-destructive and skip the file
    return true, "conflicting file, unhandled case"
}

func ensureNoRemoteModifications(files []*changedFile) error {
    conflicts := findRemoteConflicts(files)
    if len(conflicts) == 0 {
        return nil
    }

    buffer := bytes.NewBufferString("")
    formatConflicts(conflicts, buffer)
    return fmt.Errorf(buffer.String())
}
<commit_msg>Return result from recursive call<commit_after>package drive

import (
    "fmt"
    "io"
    "os"
    "time"
    "sort"
    "bytes"
    "path/filepath"
    "google.golang.org/api/googleapi"
    "google.golang.org/api/drive/v3"
)

type UploadSyncArgs struct {
    Out io.Writer
    Progress io.Writer
    Path string
    RootId string
    DryRun bool
    DeleteExtraneous bool
    ChunkSize int64
    Resolution ConflictResolution
    Comparer FileComparer
}

func (self *Drive) UploadSync(args UploadSyncArgs) error {
    if args.ChunkSize > intMax() - 1 {
        return fmt.Errorf("Chunk size is to big, max chunk size for this computer is %d", intMax() - 1)
    }

    fmt.Fprintln(args.Out, "Starting sync...")
    started := time.Now()

    // Create root directory if it does not exist
    rootDir, err := self.prepareSyncRoot(args)
    if err != nil {
        return err
    }

    fmt.Fprintln(args.Out, "Collecting local and remote file information...")
    files, err := self.prepareSyncFiles(args.Path, rootDir, args.Comparer)
    if err != nil {
        return err
    }

    // Find changed files
    changedFiles := files.filterChangedLocalFiles()

    fmt.Fprintf(args.Out, "Found %d local files and %d remote files\n", len(files.local), len(files.remote))

    // Ensure that that we don't overwrite any remote changes
    if args.Resolution == NoResolution {
        err = ensureNoRemoteModifications(changedFiles)
        if err != nil {
            return fmt.Errorf("Conflict detected!\nThe following files have changed and the remote file are newer than it's local counterpart:\n\n%s\nNo conflict resolution was given, aborting...", err)
        }
    }

    // Create missing directories
    files, err = self.createMissingRemoteDirs(files, args)
    if err != nil {
        return err
    }

    // Upload missing files
    err = self.uploadMissingFiles(files, args)
    if err != nil {
        return err
    }

    // Update modified files
    err = self.updateChangedFiles(changedFiles, rootDir, args)
    if err != nil {
        return err
    }

    // Delete extraneous files on drive
    if args.DeleteExtraneous {
        err = self.deleteExtraneousRemoteFiles(files, args)
        if err != nil {
            return err
        }
    }
    fmt.Fprintf(args.Out, "Sync finished in %s\n", time.Since(started))

    return nil
}

func (self *Drive) prepareSyncRoot(args UploadSyncArgs) (*drive.File, error) {
    fields := []googleapi.Field{"id", "name", "mimeType", "appProperties"}
    f, err := self.service.Files.Get(args.RootId).Fields(fields...).Do()
    if err != nil {
        return nil, fmt.Errorf("Failed to find root dir: %s", err)
    }

    // Ensure file is a directory
    if !isDir(f) {
        return nil, fmt.Errorf("Provided root id is not a directory")
    }

    // Return directory if syncRoot property is already set
    if _, ok := f.AppProperties["isSyncRoot"]; ok {
        return f, nil
    }

    // This is the first time this directory have been used for sync
    // Check if the directory is empty
    isEmpty, err := self.dirIsEmpty(f.Id)
    if err != nil {
        return nil, fmt.Errorf("Failed to check if root dir is empty: %s", err)
    }

    // Ensure that the directory is empty
    if !isEmpty {
        return nil, fmt.Errorf("Root directoy is not empty, the initial sync requires an empty directory")
    }

    // Update directory with syncRoot property
    dstFile := &drive.File{
        AppProperties: map[string]string{"isSyncRoot": "true"},
    }

    f, err = self.service.Files.Update(f.Id, dstFile).Fields(fields...).Do()
    if err != nil {
        return nil, fmt.Errorf("Failed to update root directory: %s", err)
    }

    return f, nil
}

func (self *Drive) createMissingRemoteDirs(files *syncFiles, args UploadSyncArgs) (*syncFiles, error) {
    missingDirs := files.filterMissingRemoteDirs()
    missingCount := len(missingDirs)

    if missingCount > 0 {
        fmt.Fprintf(args.Out, "\n%d remote directories are missing\n", missingCount)
    }

    // Sort directories so that the dirs with the shortest path comes first
    sort.Sort(byLocalPathLength(missingDirs))

    for i, lf := range missingDirs {
        parentPath := parentFilePath(lf.relPath)
        parent, ok := files.findRemoteByPath(parentPath)
        if !ok {
            return nil, fmt.Errorf("Could not find remote directory with path '%s'", parentPath)
        }

        fmt.Fprintf(args.Out, "[%04d/%04d] Creating directory %s\n", i + 1, missingCount, filepath.Join(files.root.file.Name, lf.relPath))

        f, err := self.createMissingRemoteDir(createMissingRemoteDirArgs{
            name: lf.info.Name(),
            parentId: parent.file.Id,
            rootId: args.RootId,
            dryRun: args.DryRun,
            try: 0,
        })
        if err != nil {
            return nil, err
        }

        files.remote = append(files.remote, &RemoteFile{
            relPath: lf.relPath,
            file: f,
        })
    }

    return files, nil
}

type createMissingRemoteDirArgs struct {
    name string
    parentId string
    rootId string
    dryRun bool
    try int
}

func (self *Drive) uploadMissingFiles(files *syncFiles, args UploadSyncArgs) error {
    missingFiles := files.filterMissingRemoteFiles()
    missingCount := len(missingFiles)

    if missingCount > 0 {
        fmt.Fprintf(args.Out, "\n%d remote files are missing\n", missingCount)
    }

    for i, lf := range missingFiles {
        parentPath := parentFilePath(lf.relPath)
        parent, ok := files.findRemoteByPath(parentPath)
        if !ok {
            return fmt.Errorf("Could not find remote directory with path '%s'", parentPath)
        }

        fmt.Fprintf(args.Out, "[%04d/%04d] Uploading %s -> %s\n", i + 1, missingCount, lf.relPath, filepath.Join(files.root.file.Name, lf.relPath))

        if args.DryRun {
            continue
        }

        err := self.uploadMissingFile(parent.file.Id, lf, args, 0)
        if err != nil {
            return err
        }
    }

    return nil
}

func (self *Drive) updateChangedFiles(changedFiles []*changedFile, root *drive.File, args UploadSyncArgs) error {
    changedCount := len(changedFiles)

    if changedCount > 0 {
        fmt.Fprintf(args.Out, "\n%d local files has changed\n", changedCount)
    }

    for i, cf := range changedFiles {
        if skip, reason := checkRemoteConflict(cf, args.Resolution); skip {
            fmt.Fprintf(args.Out, "[%04d/%04d] Skipping %s (%s)\n", i + 1, changedCount, cf.local.relPath, reason)
            continue
        }

        fmt.Fprintf(args.Out, "[%04d/%04d] Updating %s -> %s\n", i + 1, changedCount, cf.local.relPath, filepath.Join(root.Name, cf.local.relPath))

        if args.DryRun {
            continue
        }

        err := self.updateChangedFile(cf, args, 0)
        if err != nil {
            return err
        }
    }

    return nil
}

func (self *Drive) deleteExtraneousRemoteFiles(files *syncFiles, args UploadSyncArgs) error {
    extraneousFiles := files.filterExtraneousRemoteFiles()
    extraneousCount := len(extraneousFiles)

    if extraneousCount > 0 {
        fmt.Fprintf(args.Out, "\n%d remote files are extraneous\n", extraneousCount)
    }

    // Sort files so that the files with the longest path comes first
    sort.Sort(sort.Reverse(byRemotePathLength(extraneousFiles)))

    for i, rf := range extraneousFiles {
        fmt.Fprintf(args.Out, "[%04d/%04d] Deleting %s\n", i + 1, extraneousCount, filepath.Join(files.root.file.Name, rf.relPath))

        if args.DryRun {
            continue
        }

        err := self.deleteRemoteFile(rf, args, 0)
        if err != nil {
            return err
        }
    }

    return nil
}

func (self *Drive) createMissingRemoteDir(args createMissingRemoteDirArgs) (*drive.File, error) {
    dstFile := &drive.File{
        Name: args.name,
        MimeType: DirectoryMimeType,
        Parents: []string{args.parentId},
        AppProperties: map[string]string{"syncRootId": args.rootId},
    }

    if args.dryRun {
        return dstFile, nil
    }

    f, err := self.service.Files.Create(dstFile).Do()
    if err != nil {
        if isBackendError(err) && args.try < MaxBackendErrorRetries {
            exponentialBackoffSleep(args.try)
            args.try++
            return self.createMissingRemoteDir(args)
        } else {
            return nil, fmt.Errorf("Failed to create directory: %s", err)
        }
    }

    return f, nil
}

func (self *Drive) uploadMissingFile(parentId string, lf *LocalFile, args UploadSyncArgs, try int) error {
    srcFile, err := os.Open(lf.absPath)
    if err != nil {
        return fmt.Errorf("Failed to open file: %s", err)
    }

    // Close file on function exit
    defer srcFile.Close()

    // Instantiate drive file
    dstFile := &drive.File{
        Name: lf.info.Name(),
        Parents: []string{parentId},
        AppProperties: map[string]string{"syncRootId": args.RootId},
    }

    // Chunk size option
    chunkSize := googleapi.ChunkSize(int(args.ChunkSize))

    // Wrap file in progress reader
    srcReader := getProgressReader(srcFile, args.Progress, lf.info.Size())

    _, err = self.service.Files.Create(dstFile).Fields("id", "name", "size", "md5Checksum").Media(srcReader, chunkSize).Do()
    if err != nil {
        if isBackendError(err) && try < MaxBackendErrorRetries {
            exponentialBackoffSleep(try)
            try++
            return self.uploadMissingFile(parentId, lf, args, try)
        } else {
            return fmt.Errorf("Failed to upload file: %s", err)
        }
    }

    return nil
}

func (self *Drive) updateChangedFile(cf *changedFile, args UploadSyncArgs, try int) error {
    srcFile, err := os.Open(cf.local.absPath)
    if err != nil {
        return fmt.Errorf("Failed to open file: %s", err)
    }

    // Close file on function exit
    defer srcFile.Close()

    // Instantiate drive file
    dstFile := &drive.File{}

    // Chunk size option
    chunkSize := googleapi.ChunkSize(int(args.ChunkSize))

    // Wrap file in progress reader
    srcReader := getProgressReader(srcFile, args.Progress, cf.local.info.Size())

    _, err = self.service.Files.Update(cf.remote.file.Id, dstFile).Media(srcReader, chunkSize).Do()
    if err != nil {
        if isBackendError(err) && try < MaxBackendErrorRetries {
            exponentialBackoffSleep(try)
            try++
            return self.updateChangedFile(cf, args, try)
        } else {
            return fmt.Errorf("Failed to update file: %s", err)
        }
    }

    return nil
}

func (self *Drive) deleteRemoteFile(rf *RemoteFile, args UploadSyncArgs, try int) error {
    err := self.service.Files.Delete(rf.file.Id).Do()
    if err != nil {
        if isBackendError(err) && try < MaxBackendErrorRetries {
            exponentialBackoffSleep(try)
            try++
            return self.deleteRemoteFile(rf, args, try)
        } else {
            return fmt.Errorf("Failed to delete file: %s", err)
        }
    }

    return nil
}

func (self *Drive) dirIsEmpty(id string) (bool, error) {
    query := fmt.Sprintf("'%s' in parents", id)
    fileList, err := self.service.Files.List().Q(query).Do()
    if err != nil {
        return false, fmt.Errorf("Empty dir check failed: ", err)
    }

    return len(fileList.Files) == 0, nil
}

func checkRemoteConflict(cf *changedFile, resolution ConflictResolution) (bool, string) {
    // No conflict unless remote file was last modified
    if cf.compareModTime() != RemoteLastModified {
        return false, ""
    }

    // Don't skip if want to keep the local file
    if resolution == KeepLocal {
        return false, ""
    }

    // Skip if we want to keep the remote file
    if resolution == KeepRemote {
        return true, "conflicting file, keeping remote file"
    }

    if resolution == KeepLargest {
        largest := cf.compareSize()

        // Skip if the remote file is largest
        if largest == RemoteLargestSize {
            return true, "conflicting file, remote file is largest, keeping remote"
        }

        // Don't skip if the local file is largest
        if largest == LocalLargestSize {
            return false, ""
        }

        // Keep remote if both files have the same size
        if largest == EqualSize {
            return true, "conflicting file, file sizes are equal, keeping remote"
        }
    }

    // The conditionals above should cover all cases,
    // unless the programmer did something wrong,
    // in which case we default to being non-destructive and skip the file
    return true, "conflicting file, unhandled case"
}

func ensureNoRemoteModifications(files []*changedFile) error {
    conflicts := findRemoteConflicts(files)
    if len(conflicts) == 0 {
        return nil
    }

    buffer := bytes.NewBufferString("")
    formatConflicts(conflicts, buffer)
    return fmt.Errorf(buffer.String())
}
<|endoftext|>


<commit_before>package DefaultLogger

import (
	"fmt"
	"github.com/ian-kent/go-log/appenders"
	"github.com/ian-kent/go-log/layout"
	"github.com/ian-kent/go-log/levels"
	golog "github.com/ian-kent/go-log/log"
	gologger "github.com/ian-kent/go-log/logger"
	"time"

	. "github.com/francoishill/golang-common-ddd/Interface/Logger"
)

type logger struct {
	logger gologger.Logger
}

func (l *logger) formatNowTime() string {
	now := time.Now()
	_, offset := now.Zone()
	var timezoneSign string
	if offset >= 0 {
		timezoneSign = "+"
	} else {
		timezoneSign = "-"
	}
	return fmt.Sprintf("%s %s%d", now.Local().Format("2006-01-02 15:04:05"), timezoneSign, offset/(60*60))
}

func (l *logger) combineParams(msg string, params ...interface{}) []interface{} {
	combined := []interface{}{
		fmt.Sprintf("{%s} %s", l.formatNowTime(), msg),
	}
	combined = append(combined, params...)
	return combined
}

func (l *logger) Emergency(msg string, params ...interface{}) {
	l.logger.Fatal(l.combineParams(msg, params...)...)
}
func (l *logger) Alert(msg string, params ...interface{}) {
	l.logger.Error(l.combineParams(msg, params...)...)
}
func (l *logger) Critical(msg string, params ...interface{}) {
	l.logger.Error(l.combineParams(msg, params...)...)
}
func (l *logger) Error(msg string, params ...interface{}) {
	l.logger.Error(l.combineParams(msg, params...)...)
}
func (l *logger) Warn(msg string, params ...interface{}) {
	l.logger.Warn(l.combineParams(msg, params...)...)
}
func (l *logger) Notice(msg string, params ...interface{}) {
	l.logger.Info(l.combineParams(msg, params...)...)
}
func (l *logger) Info(msg string, params ...interface{}) {
	l.logger.Info(l.combineParams(msg, params...)...)
}
func (l *logger) Debug(msg string, params ...interface{}) {
	l.logger.Debug(l.combineParams(msg, params...)...)
}

func getPrefixWithSpace(prefix string) string {
	if prefix == "" {
		return ""
	} else {
		return prefix + " "
	}
}

func New(logFileName, prefix string, isDevMode bool) Logger {
	l := golog.Logger()

	if isDevMode {
		l.SetLevel(levels.DEBUG)
	} else {
		l.SetLevel(levels.INFO)
	}

	layoutToUse := layout.Pattern(getPrefixWithSpace(prefix) + "[%p] %m") //level/priority, message

	rollingFileAppender := appenders.RollingFile(logFileName, true)
	rollingFileAppender.MaxBackupIndex = 5
	rollingFileAppender.MaxFileSize = 20 * 1024 * 1024 // 20 MB
	rollingFileAppender.SetLayout(layoutToUse)

	consoleAppender := appenders.Console()
	consoleAppender.SetLayout(layoutToUse)
	l.SetAppender(
		Multiple( //appenders.Multiple( ONCE PULL REQUEST OF ABOVE IS IN
			layoutToUse,
			rollingFileAppender,
			consoleAppender,
		))

	return &logger{l}
}
<commit_msg>Added escaping of newline chars in DefaultLogger: This will just escape the '\r' with '' '\n' with '\\n'.<commit_after>package DefaultLogger

import (
	"fmt"
	"github.com/ian-kent/go-log/appenders"
	"github.com/ian-kent/go-log/layout"
	"github.com/ian-kent/go-log/levels"
	golog "github.com/ian-kent/go-log/log"
	gologger "github.com/ian-kent/go-log/logger"
	"strings"
	"time"

	. "github.com/francoishill/golang-common-ddd/Interface/Logger"
)

type logger struct {
	logger gologger.Logger
}

func (l *logger) formatNowTime() string {
	now := time.Now()
	_, offset := now.Zone()
	var timezoneSign string
	if offset >= 0 {
		timezoneSign = "+"
	} else {
		timezoneSign = "-"
	}
	return fmt.Sprintf("%s %s%d", now.Local().Format("2006-01-02 15:04:05"), timezoneSign, offset/(60*60))
}

func (l *logger) escapeNewlinesChars(s string) string {
	r := strings.NewReplacer("\r", "", "\n", "\\n")
	return r.Replace(s)
}

func (l *logger) combineParams(msg string, params ...interface{}) []interface{} {
	combined := []interface{}{
		fmt.Sprintf("{%s} %s", l.formatNowTime(), l.escapeNewlinesChars(msg)),
	}
	combined = append(combined, params...)
	return combined
}

func (l *logger) Emergency(msg string, params ...interface{}) {
	l.logger.Fatal(l.combineParams(msg, params...)...)
}
func (l *logger) Alert(msg string, params ...interface{}) {
	l.logger.Error(l.combineParams(msg, params...)...)
}
func (l *logger) Critical(msg string, params ...interface{}) {
	l.logger.Error(l.combineParams(msg, params...)...)
}
func (l *logger) Error(msg string, params ...interface{}) {
	l.logger.Error(l.combineParams(msg, params...)...)
}
func (l *logger) Warn(msg string, params ...interface{}) {
	l.logger.Warn(l.combineParams(msg, params...)...)
}
func (l *logger) Notice(msg string, params ...interface{}) {
	l.logger.Info(l.combineParams(msg, params...)...)
}
func (l *logger) Info(msg string, params ...interface{}) {
	l.logger.Info(l.combineParams(msg, params...)...)
}
func (l *logger) Debug(msg string, params ...interface{}) {
	l.logger.Debug(l.combineParams(msg, params...)...)
}

func getPrefixWithSpace(prefix string) string {
	if prefix == "" {
		return ""
	} else {
		return prefix + " "
	}
}

func New(logFileName, prefix string, isDevMode bool) Logger {
	l := golog.Logger()

	if isDevMode {
		l.SetLevel(levels.DEBUG)
	} else {
		l.SetLevel(levels.INFO)
	}

	layoutToUse := layout.Pattern(getPrefixWithSpace(prefix) + "[%p] %m") //level/priority, message

	rollingFileAppender := appenders.RollingFile(logFileName, true)
	rollingFileAppender.MaxBackupIndex = 5
	rollingFileAppender.MaxFileSize = 20 * 1024 * 1024 // 20 MB
	rollingFileAppender.SetLayout(layoutToUse)

	consoleAppender := appenders.Console()
	consoleAppender.SetLayout(layoutToUse)
	l.SetAppender(
		Multiple( //appenders.Multiple( ONCE PULL REQUEST OF ABOVE IS IN
			layoutToUse,
			rollingFileAppender,
			consoleAppender,
		))

	return &logger{l}
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

// Package client implements OAuth2 authentication for outbound connections
// from Appengine using the application services account.
package client

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"golang.org/x/net/context"

	"github.com/luci/gae/service/info"

	"github.com/luci/luci-go/common/auth"
	"github.com/luci/luci-go/common/clock"
	"github.com/luci/luci-go/common/data/caching/proccache"
	"github.com/luci/luci-go/common/data/rand/mathrand"
	"github.com/luci/luci-go/common/data/stringset"
	"github.com/luci/luci-go/common/errors"
	"github.com/luci/luci-go/common/logging"
)

// GetAccessToken returns an OAuth access token representing app's service
// account.
//
// If scopes is empty, uses auth.OAuthScopeEmail scope.
//
// Implements a caching layer on top of GAE's GetAccessToken RPC. May return
// transient errors.
func GetAccessToken(c context.Context, scopes []string) (auth.Token, error) {
	scopes, cacheKey := normalizeScopes(scopes)

	// Try to find the token in the local memory first. If it expires soon,
	// refresh it earlier with some probability. That avoids a situation when
	// parallel requests that use access tokens suddenly see the cache expired
	// and rush to refresh the token all at once.
	pcache := proccache.GetCache(c)
	if entry := pcache.Get(cacheKey); entry != nil && !closeToExpRandomized(c, entry.Exp) {
		return entry.Value.(auth.Token), nil
	}

	// The token needs to be refreshed.
	logging.Debugf(c, "Getting an access token for scopes %q", strings.Join(scopes, ", "))
	accessToken, exp, err := info.AccessToken(c, scopes...)
	if err != nil {
		return auth.Token{}, errors.WrapTransient(err)
	}

	// Prematurely expire it to guarantee all returned token live for at least
	// 'expirationMinLifetime'.
	tok := auth.Token{
		AccessToken: accessToken,
		Expiry:      exp.Add(-expirationMinLifetime),
		TokenType:   "Bearer",
	}

	// Store the new token in the cache (overriding what's already there).
	pcache.Put(cacheKey, tok, tok.Expiry)

	return tok, nil
}

//// Internal stuff.

type cacheKey string

const (
	// expirationMinLifetime is minimal possible lifetime of a returned token.
	expirationMinLifetime = 2 * time.Minute
	// expirationRandomization defines how much to randomize expiration time.
	expirationRandomization = 10 * time.Minute
)

func normalizeScopes(scopes []string) ([]string, cacheKey) {
	if len(scopes) == 0 {
		scopes = []string{auth.OAuthScopeEmail}
	} else {
		set := stringset.New(len(scopes))
		for _, s := range scopes {
			if strings.ContainsRune(s, '\n') {
				panic(fmt.Errorf("invalid scope %q", s))
			}
			set.Add(s)
		}
		scopes = set.ToSlice()
		sort.Strings(scopes)
	}
	return scopes, cacheKey(strings.Join(scopes, "\n"))
}

func closeToExpRandomized(c context.Context, exp time.Time) bool {
	switch now := clock.Now(c); {
	case now.After(exp):
		return true // expired already
	case now.Add(expirationRandomization).Before(exp):
		return false // far from expiration
	default:
		// The expiration is close enough. Do the randomization.
		rnd := time.Duration(mathrand.Get(c).Int63n(int64(expirationRandomization)))
		return now.Add(rnd).After(exp)
	}
}
<commit_msg>Log GAE service account access token expiration time.<commit_after>// Copyright 2015 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

// Package client implements OAuth2 authentication for outbound connections
// from Appengine using the application services account.
package client

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"golang.org/x/net/context"

	"github.com/luci/gae/service/info"

	"github.com/luci/luci-go/common/auth"
	"github.com/luci/luci-go/common/clock"
	"github.com/luci/luci-go/common/data/caching/proccache"
	"github.com/luci/luci-go/common/data/rand/mathrand"
	"github.com/luci/luci-go/common/data/stringset"
	"github.com/luci/luci-go/common/errors"
	"github.com/luci/luci-go/common/logging"
)

// GetAccessToken returns an OAuth access token representing app's service
// account.
//
// If scopes is empty, uses auth.OAuthScopeEmail scope.
//
// Implements a caching layer on top of GAE's GetAccessToken RPC. May return
// transient errors.
func GetAccessToken(c context.Context, scopes []string) (auth.Token, error) {
	scopes, cacheKey := normalizeScopes(scopes)

	// Try to find the token in the local memory first. If it expires soon,
	// refresh it earlier with some probability. That avoids a situation when
	// parallel requests that use access tokens suddenly see the cache expired
	// and rush to refresh the token all at once.
	pcache := proccache.GetCache(c)
	if entry := pcache.Get(cacheKey); entry != nil && !closeToExpRandomized(c, entry.Exp) {
		return entry.Value.(auth.Token), nil
	}

	// The token needs to be refreshed.
	logging.Debugf(c, "Getting an access token for scopes %q", strings.Join(scopes, ", "))
	accessToken, exp, err := info.AccessToken(c, scopes...)
	if err != nil {
		return auth.Token{}, errors.WrapTransient(err)
	}
	logging.Debugf(c, "The token expires in %s", exp.Sub(clock.Now(c)))

	// Prematurely expire it to guarantee all returned token live for at least
	// 'expirationMinLifetime'.
	tok := auth.Token{
		AccessToken: accessToken,
		Expiry:      exp.Add(-expirationMinLifetime),
		TokenType:   "Bearer",
	}

	// Store the new token in the cache (overriding what's already there).
	pcache.Put(cacheKey, tok, tok.Expiry)

	return tok, nil
}

//// Internal stuff.

type cacheKey string

const (
	// expirationMinLifetime is minimal possible lifetime of a returned token.
	expirationMinLifetime = 2 * time.Minute
	// expirationRandomization defines how much to randomize expiration time.
	expirationRandomization = 10 * time.Minute
)

func normalizeScopes(scopes []string) ([]string, cacheKey) {
	if len(scopes) == 0 {
		scopes = []string{auth.OAuthScopeEmail}
	} else {
		set := stringset.New(len(scopes))
		for _, s := range scopes {
			if strings.ContainsRune(s, '\n') {
				panic(fmt.Errorf("invalid scope %q", s))
			}
			set.Add(s)
		}
		scopes = set.ToSlice()
		sort.Strings(scopes)
	}
	return scopes, cacheKey(strings.Join(scopes, "\n"))
}

func closeToExpRandomized(c context.Context, exp time.Time) bool {
	switch now := clock.Now(c); {
	case now.After(exp):
		return true // expired already
	case now.Add(expirationRandomization).Before(exp):
		return false // far from expiration
	default:
		// The expiration is close enough. Do the randomization.
		rnd := time.Duration(mathrand.Get(c).Int63n(int64(expirationRandomization)))
		return now.Add(rnd).After(exp)
	}
}
<|endoftext|>


<commit_before>package linenotify

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"time"

	"github.com/pkg/errors"
	"github.com/utahta/go-linenotify"
	"github.com/utahta/momoclo-channel/appengine/lib/log"
	"github.com/utahta/momoclo-channel/appengine/model"
	"github.com/utahta/momoclo-crawler"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
	"google.golang.org/appengine/urlfetch"
)

const timeout = 540 * time.Second

// Send message to LINE Notify
func NotifyMessage(ctx context.Context, message string) error {
	ctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	c, err := newClient(ctx)
	if err != nil {
		return err
	}

	// [Notify Name] が付くので先頭に改行をいれて調整
	return c.notifyMessage(fmt.Sprintf("\n%s", message), "")
}

// Send channel message and images to LINE Notify
func NotifyChannel(ctx context.Context, ch *crawler.Channel) error {
	ctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	c, err := newClient(ctx)
	if err != nil {
		return err
	}

	eg := &errgroup.Group{}
	for _, item := range ch.Items {
		item := item
		eg.Go(func() error {
			if err := model.NewLineItem(item).Put(ctx); err != nil {
				return nil
			}
			if err := c.notifyChannelItem(ch.Title, item); err != nil {
				log.GaeLog(ctx).Error(err)
				return err
			}
			return nil
		})
	}

	if err := eg.Wait(); err != nil {
		return errors.New("Errors occurred in NotifyChannel")
	}
	return nil
}

type client struct {
	*linenotify.Client
	users   []*model.LineNotification
	context context.Context
	log     log.Logger
}

func newClient(ctx context.Context) (*client, error) {
	c := &client{
		Client:  linenotify.New(),
		context: ctx,
		log:     log.NewGaeLogger(ctx),
	}
	c.HTTPClient.Transport = &urlfetch.Transport{Context: ctx}

	query := model.NewLineNotificationQuery(ctx)
	users, err := query.GetAll()
	if err != nil {
		return nil, err
	}
	c.users = users

	return c, nil
}

func (c *client) notifyChannelItem(title string, item *crawler.ChannelItem) error {
	message := fmt.Sprintf("\n%s\n%s\n%s", title, item.Title, item.Url)

	if len(item.Images) > 0 {
		image := item.Images[0]
		if err := c.notifyMessage(message, image.Url); err != nil {
			return err
		}

		for _, image := range item.Images[1:] {
			if err := c.notifyMessage(" ", image.Url); err != nil {
				return err
			}
		}
	} else {
		return c.notifyMessage(message, "")
	}
	return nil
}

func (c *client) notifyMessage(message, imageURL string) error {
	if disabled() {
		return nil
	}

	// prepare cached image
	if imageURL != "" {
		_, err := fetchImage(c.HTTPClient, imageURL)
		if err != nil {
			return err
		}
		defer clearImage(imageURL)
	}

	var workQueue = make(chan bool, 100) // max goroutine
	eg := &errgroup.Group{}
	for _, user := range c.users {
		user := user
		workQueue <- true

		eg.Go(func() error {
			defer func() {
				<-workQueue
			}()

			token, err := user.Token()
			if err != nil {
				c.log.Errorf("Failed to get token. hash:%v err:%v", user.Id, err)
				return err
			}

			var image io.Reader
			if b := cacheImage(imageURL); b != nil {
				image = bytes.NewReader(b)
			}

			err = c.Notify(token, message, "", "", image)
			if err == linenotify.ErrNotifyInvalidAccessToken {
				user.Delete(c.context)
				c.log.Infof("Delete LINE Notify token. hash:%s", user.Id)
			} else if err != nil {
				c.log.Error("Failed to notify. hash:%v err:%v", user.Id, err)
				return err
			}
			return nil
		})
	}

	if err := eg.Wait(); err != nil {
		c.log.Warningf("Errors occurred in notifyMessage.")
	}
	c.log.Infof("LINE Notify. message:%s imageURL:%s len:%d", message, imageURL, len(c.users))
	return nil
}

// if true disable linenotify
func disabled() bool {
	e := os.Getenv("LINENOTIFY_DISABLE")
	if e != "" {
		return true
	}
	return false
}
<commit_msg>Fix format<commit_after>package linenotify

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"time"

	"github.com/pkg/errors"
	"github.com/utahta/go-linenotify"
	"github.com/utahta/momoclo-channel/appengine/lib/log"
	"github.com/utahta/momoclo-channel/appengine/model"
	"github.com/utahta/momoclo-crawler"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
	"google.golang.org/appengine/urlfetch"
)

const timeout = 540 * time.Second

// Send message to LINE Notify
func NotifyMessage(ctx context.Context, message string) error {
	ctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	c, err := newClient(ctx)
	if err != nil {
		return err
	}

	// [Notify Name] が付くので先頭に改行をいれて調整
	return c.notifyMessage(fmt.Sprintf("\n%s", message), "")
}

// Send channel message and images to LINE Notify
func NotifyChannel(ctx context.Context, ch *crawler.Channel) error {
	ctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	c, err := newClient(ctx)
	if err != nil {
		return err
	}

	eg := &errgroup.Group{}
	for _, item := range ch.Items {
		item := item
		eg.Go(func() error {
			if err := model.NewLineItem(item).Put(ctx); err != nil {
				return nil
			}
			if err := c.notifyChannelItem(ch.Title, item); err != nil {
				log.GaeLog(ctx).Error(err)
				return err
			}
			return nil
		})
	}

	if err := eg.Wait(); err != nil {
		return errors.New("Errors occurred in NotifyChannel")
	}
	return nil
}

type client struct {
	*linenotify.Client
	users   []*model.LineNotification
	context context.Context
	log     log.Logger
}

func newClient(ctx context.Context) (*client, error) {
	c := &client{
		Client:  linenotify.New(),
		context: ctx,
		log:     log.NewGaeLogger(ctx),
	}
	c.HTTPClient.Transport = &urlfetch.Transport{Context: ctx}

	query := model.NewLineNotificationQuery(ctx)
	users, err := query.GetAll()
	if err != nil {
		return nil, err
	}
	c.users = users

	return c, nil
}

func (c *client) notifyChannelItem(title string, item *crawler.ChannelItem) error {
	message := fmt.Sprintf("\n%s\n%s\n%s", title, item.Title, item.Url)

	if len(item.Images) > 0 {
		image := item.Images[0]
		if err := c.notifyMessage(message, image.Url); err != nil {
			return err
		}

		for _, image := range item.Images[1:] {
			if err := c.notifyMessage(" ", image.Url); err != nil {
				return err
			}
		}
	} else {
		return c.notifyMessage(message, "")
	}
	return nil
}

func (c *client) notifyMessage(message, imageURL string) error {
	if disabled() {
		return nil
	}

	// prepare cached image
	if imageURL != "" {
		_, err := fetchImage(c.HTTPClient, imageURL)
		if err != nil {
			return err
		}
		defer clearImage(imageURL)
	}

	var workQueue = make(chan bool, 100) // max goroutine
	eg := &errgroup.Group{}
	for _, user := range c.users {
		user := user
		workQueue <- true

		eg.Go(func() error {
			defer func() {
				<-workQueue
			}()

			token, err := user.Token()
			if err != nil {
				c.log.Errorf("Failed to get token. hash:%v err:%v", user.Id, err)
				return err
			}

			var image io.Reader
			if b := cacheImage(imageURL); b != nil {
				image = bytes.NewReader(b)
			}

			err = c.Notify(token, message, "", "", image)
			if err == linenotify.ErrNotifyInvalidAccessToken {
				user.Delete(c.context)
				c.log.Infof("Delete LINE Notify token. hash:%s", user.Id)
			} else if err != nil {
				c.log.Errorf("Failed to notify. hash:%v err:%v", user.Id, err)
				return err
			}
			return nil
		})
	}

	if err := eg.Wait(); err != nil {
		c.log.Warningf("Errors occurred in notifyMessage.")
	}
	c.log.Infof("LINE Notify. message:%s imageURL:%s len:%d", message, imageURL, len(c.users))
	return nil
}

// if true disable linenotify
func disabled() bool {
	e := os.Getenv("LINENOTIFY_DISABLE")
	if e != "" {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"launchpad.net/loggo"

	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/juju"
	_ "launchpad.net/juju-core/provider/all"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/utils"
)

var logger = loggo.GetLogger("juju.plugins.updatebootstrap")

const updateBootstrapDoc = `
Patches all machines after state server has been restored from backup, to
update state server address to new location.
`

type updateBootstrapCommand struct {
	cmd.EnvCommandBase
}

func (c *updateBootstrapCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "juju-update-bootstrap",
		Purpose: "update all machines after recovering state server",
		Doc:     updateBootstrapDoc,
	}
}

func (c *updateBootstrapCommand) Run(ctx *cmd.Context) error {
	conn, err := juju.NewConnFromName(c.EnvName)
	if err != nil {
		return err
	}
	defer conn.Close()
	stateAddr, err := GetStateAddress(conn.Environ)
	if err != nil {
		return err
	}
	fmt.Printf("using state address %v\n", stateAddr)
	return updateAllMachines(conn, stateAddr)
}

// GetStateAddress returns the address of one state server
func GetStateAddress(environ environs.Environ) (string, error) {
	// XXX: Can easily look up state server address using api instead
	stateInfo, _, err := environ.StateInfo()
	if err != nil {
		return "", err
	}
	return strings.Split(stateInfo.Addrs[0], ":")[0], nil
}

var agentAddressTemplate = `
set -exu
cd /var/lib/juju/agents
for agent in *
do
	initctl stop jujud-$agent
	sed -i.old -r "/^(stateaddresses|apiaddresses):/{
		n
		s/- .*(:[0-9]+)/- $ADDR\1/
	}" $agent/agent.conf
	if [[ $agent = unit-* ]]
	then
		sed -i -r 's/change-version: [0-9]+$/change-version: 0/' $agent/state/relations/*/*
	fi
	initctl start jujud-$agent
done
sed -i -r 's/^(:syslogtag, startswith, "juju-" @)(.*)(:[0-9]+)$/\1'$ADDR'\3/' /etc/rsyslog.d/*-juju*.conf
`

// renderScriptArg generates an ssh script argument to update state addresses
func renderScriptArg(stateAddr string) string {
	script := strings.Replace(agentAddressTemplate, "$ADDR", stateAddr, -1)
	return "sudo bash -c " + utils.ShQuote(script)
}

// runMachineUpdate connects via ssh to the machine and runs the update script
func runMachineUpdate(m *state.Machine, sshArg string) error {
	logger.Infof("updating machine: %v\n", m)
	addr := instance.SelectPublicAddress(m.Addresses())
	if addr == "" {
		return fmt.Errorf("no appropriate public address found")
	}
	args := []string{
		"-l", "ubuntu",
		"-T",
		"-o", "StrictHostKeyChecking no",
		"-o", "PasswordAuthentication no",
		addr,
		sshArg,
	}
	c := exec.Command("ssh", args...)
	if data, err := c.CombinedOutput(); err != nil {
		return fmt.Errorf("ssh command failed: %v (%q)", err, data)
	}
	return nil
}

// updateAllMachines finds all machines resets the stored state address
func updateAllMachines(conn *juju.Conn, stateAddr string) error {
	machines, err := conn.State.AllMachines()
	if err != nil {
		return err
	}
	pendingMachineCount := 0
	done := make(chan error)
	for _, machine := range machines {
		// A newly resumed state server requires no updating, and more
		// than one state server is not yet support by this plugin.
		if machine.IsStateServer() {
			continue
		}
		pendingMachineCount += 1
		machine := machine
		go func() {
			err := runMachineUpdate(machine, renderScriptArg(stateAddr))
			if err != nil {
				logger.Errorf("failed to update machine %s: %v", machine, err)
			} else {
				logger.Infof("updated machine %s", machine)
			}
			done <- err
		}()
	}
	err = nil
	for ; pendingMachineCount > 0; pendingMachineCount-- {
		if updateErr := <-done; updateErr != nil && err == nil {
			err = fmt.Errorf("machine update failed")
		}
	}
	return err
}

func Main(args []string) {
	if err := juju.InitJujuHome(); err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err)
		os.Exit(2)
	}
	command := updateBootstrapCommand{}
	os.Exit(cmd.Main(&command, cmd.DefaultContext(), args[1:]))
}

func main() {
	Main(os.Args)
}
<commit_msg>[r=jameinel] cmd/plugins/juju-update-bootstrap: Tweak compat<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"launchpad.net/loggo"

	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/juju"
	_ "launchpad.net/juju-core/provider/all"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/utils"
)

var logger = loggo.GetLogger("juju.plugins.updatebootstrap")

const updateBootstrapDoc = `
Patches all machines after state server has been restored from backup, to
update state server address to new location.
`

type updateBootstrapCommand struct {
	cmd.EnvCommandBase
}

func (c *updateBootstrapCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "juju-update-bootstrap",
		Purpose: "update all machines after recovering state server",
		Doc:     updateBootstrapDoc,
	}
}

func (c *updateBootstrapCommand) Run(ctx *cmd.Context) error {
	conn, err := juju.NewConnFromName(c.EnvName)
	if err != nil {
		return err
	}
	defer conn.Close()
	stateAddr, err := GetStateAddress(conn.Environ)
	if err != nil {
		return err
	}
	fmt.Printf("using state address %v\n", stateAddr)
	return updateAllMachines(conn, stateAddr)
}

// GetStateAddress returns the address of one state server
func GetStateAddress(environ environs.Environ) (string, error) {
	// XXX: Can easily look up state server address using api instead
	stateInfo, _, err := environ.StateInfo()
	if err != nil {
		return "", err
	}
	return strings.Split(stateInfo.Addrs[0], ":")[0], nil
}

var agentAddressTemplate = `
set -exu
cd /var/lib/juju/agents
for agent in *
do
	initctl stop jujud-$agent
	sed -i.old -r "/^(stateaddresses|apiaddresses):/{
		n
		s/- .*(:[0-9]+)/- $ADDR\1/
	}" $agent/agent.conf
	if [[ $agent = unit-* ]]
	then
		sed -i -r 's/change-version: [0-9]+$/change-version: 0/' $agent/state/relations/*/*
	fi
	initctl start jujud-$agent
done
sed -i -r 's/^(:syslogtag, startswith, "juju-" @)(.*)(:[0-9]+.*)$/\1'$ADDR'\3/' /etc/rsyslog.d/*-juju*.conf
`

// renderScriptArg generates an ssh script argument to update state addresses
func renderScriptArg(stateAddr string) string {
	script := strings.Replace(agentAddressTemplate, "$ADDR", stateAddr, -1)
	return "sudo bash -c " + utils.ShQuote(script)
}

// runMachineUpdate connects via ssh to the machine and runs the update script
func runMachineUpdate(m *state.Machine, sshArg string) error {
	logger.Infof("updating machine: %v\n", m)
	addr := instance.SelectPublicAddress(m.Addresses())
	if addr == "" {
		return fmt.Errorf("no appropriate public address found")
	}
	args := []string{
		"-l", "ubuntu",
		"-T",
		"-o", "StrictHostKeyChecking no",
		"-o", "PasswordAuthentication no",
		addr,
		sshArg,
	}
	c := exec.Command("ssh", args...)
	if data, err := c.CombinedOutput(); err != nil {
		return fmt.Errorf("ssh command failed: %v (%q)", err, data)
	}
	return nil
}

// updateAllMachines finds all machines resets the stored state address
func updateAllMachines(conn *juju.Conn, stateAddr string) error {
	machines, err := conn.State.AllMachines()
	if err != nil {
		return err
	}
	pendingMachineCount := 0
	done := make(chan error)
	for _, machine := range machines {
		// A newly resumed state server requires no updating, and more
		// than one state server is not yet support by this plugin.
		if machine.IsStateServer() || machine.Life() != state.Alive {
			continue
		}
		pendingMachineCount += 1
		machine := machine
		go func() {
			err := runMachineUpdate(machine, renderScriptArg(stateAddr))
			if err != nil {
				logger.Errorf("failed to update machine %s: %v", machine, err)
			} else {
				logger.Infof("updated machine %s", machine)
			}
			done <- err
		}()
	}
	err = nil
	for ; pendingMachineCount > 0; pendingMachineCount-- {
		if updateErr := <-done; updateErr != nil && err == nil {
			err = fmt.Errorf("machine update failed")
		}
	}
	return err
}

func Main(args []string) {
	if err := juju.InitJujuHome(); err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err)
		os.Exit(2)
	}
	command := updateBootstrapCommand{}
	os.Exit(cmd.Main(&command, cmd.DefaultContext(), args[1:]))
}

func main() {
	Main(os.Args)
}
<|endoftext|>


<commit_before>package test

import (
	"testing"

	"github.com/heqzha/dcache/utils"
)

func TestDCache(t *testing.T) {
	pool := utils.GetCliPoolInst()
	cli, err := pool.Get("127.0.0.1:11000")
	if err != nil {
		t.Error(err)
		return
	}
	key := "test12"
	getRes, err := cli.Get("default", key)
	if err != nil {
		t.Error(err)
		return
	}
	t.Log(string(getRes.GetValue()))
	if _, err := cli.Set("default", key, []byte("Hello World")); err != nil {
		t.Error(err)
		return
	}
	getRes, err = cli.Get("default", key)
	if err != nil {
		t.Error(err)
		return
	}
	t.Log(string(getRes.GetValue()))

	delRes, err := cli.Del("default", key)
	if err != nil {
		t.Error(err)
		return
	}
	t.Log(string(delRes.GetValue()))
}
<commit_msg>change utils to global<commit_after>package test

import (
	"testing"

	"github.com/heqzha/dcache/global"
)

func TestDCache(t *testing.T) {
	pool := global.GetCliPoolInst()
	cli, err := pool.GetOrAdd("127.0.0.1:11000")
	if err != nil {
		t.Error(err)
		return
	}
	key := "test12"
	getRes, err := cli.Get("default", key)
	if err != nil {
		t.Error(err)
		return
	}
	t.Log(string(getRes.GetValue()))
	if _, err := cli.Set("default", key, []byte("Hello World")); err != nil {
		t.Error(err)
		return
	}
	getRes, err = cli.Get("default", key)
	if err != nil {
		t.Error(err)
		return
	}
	t.Log(string(getRes.GetValue()))

	delRes, err := cli.Del("default", key)
	if err != nil {
		t.Error(err)
		return
	}
	t.Log(string(delRes.GetValue()))
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Services", func() {
	var c *client.Client

	BeforeEach(func() {
		c = loadClientOrDie()
	})

	It("should provide DNS for the cluster", func() {
		if testContext.provider == "vagrant" {
			By("Skipping test which is broken for vagrant (See https://github.com/GoogleCloudPlatform/kubernetes/issues/3580)")
		} else {
			podClient := c.Pods(api.NamespaceDefault)

			//TODO: Wait for skyDNS

			// All the names we need to be able to resolve.
			namesToResolve := []string{
				"kubernetes-ro",
				"kubernetes-ro.default",
				"kubernetes-ro.default.kubernetes.local",
				"google.com",
			}

			probeCmd := "for i in `seq 1 600`; do "
			for _, name := range namesToResolve {
				probeCmd += fmt.Sprintf("wget -O /dev/null %s && echo OK > /results/%s;", name, name)
			}
			probeCmd += "sleep 1; done"

			// Run a pod which probes DNS and exposes the results by HTTP.
			By("creating a pod to probe DNS")
			pod := &api.Pod{
				TypeMeta: api.TypeMeta{
					Kind:       "Pod",
					APIVersion: "v1beta1",
				},
				ObjectMeta: api.ObjectMeta{
					Name: "dns-test-" + string(util.NewUUID()),
				},
				Spec: api.PodSpec{
					Volumes: []api.Volume{
						{
							Name: "results",
							Source: api.VolumeSource{
								EmptyDir: &api.EmptyDir{},
							},
						},
					},
					Containers: []api.Container{
						{
							Name:  "webserver",
							Image: "kubernetes/test-webserver",
							VolumeMounts: []api.VolumeMount{
								{
									Name:      "results",
									MountPath: "/results",
								},
							},
						},
						{
							Name:    "pinger",
							Image:   "busybox",
							Command: []string{"sh", "-c", probeCmd},
							VolumeMounts: []api.VolumeMount{
								{
									Name:      "results",
									MountPath: "/results",
								},
							},
						},
					},
				},
			}

			By("submitting the pod to kuberenetes")
			_, err := podClient.Create(pod)
			if err != nil {
				Fail(fmt.Sprintf("Failed to create %s pod: %v", pod.Name, err))
			}
			defer func() {
				By("deleting the pod")
				defer GinkgoRecover()
				podClient.Delete(pod.Name)
			}()

			By("waiting for the pod to start running")
			waitForPodRunning(c, pod.Name)

			By("retrieving the pod")
			pod, err = podClient.Get(pod.Name)
			if err != nil {
				Fail(fmt.Sprintf("Failed to get pod %s: %v", pod.Name, err))
			}

			// Try to find results for each expected name.
			By("looking for the results for each expected name")
			var failed []string
			for try := 1; try < 100; try++ {
				failed = []string{}
				for _, name := range namesToResolve {
					_, err := c.Get().
						Prefix("proxy").
						Resource("pods").
						Namespace("default").
						Name(pod.Name).
						Suffix("results", name).
						Do().Raw()
					if err != nil {
						failed = append(failed, name)
						fmt.Printf("Lookup using %s for %s failed: %v\n", pod.Name, name, err)
					}
				}
				if len(failed) == 0 {
					break
				}
				fmt.Printf("lookups using %s failed for: %v\n", pod.Name, failed)
				time.Sleep(10 * time.Second)
			}
			Expect(len(failed)).To(Equal(0))

			// TODO: probe from the host, too.

			fmt.Printf("DNS probes using %s succeeded\n", pod.Name)
		}
	})

	It("should provide RW and RO services", func() {
		svc := api.ServiceList{}
		err := c.Get().
			Namespace("default").
			AbsPath("/api/v1beta1/proxy/services/kubernetes-ro/api/v1beta1/services").
			Do().
			Into(&svc)
		if err != nil {
			Fail(fmt.Sprintf("unexpected error listing services using ro service: %v", err))
		}
		var foundRW, foundRO bool
		for i := range svc.Items {
			if svc.Items[i].Name == "kubernetes" {
				foundRW = true
			}
			if svc.Items[i].Name == "kubernetes-ro" {
				foundRO = true
			}
		}
		Expect(foundRW).To(Equal(true))
		Expect(foundRO).To(Equal(true))
	})
})
<commit_msg>Changed to return in the success case testing testContext instead of using an else statement.  #4162<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Services", func() {
	var c *client.Client

	BeforeEach(func() {
		c = loadClientOrDie()
	})

	It("should provide DNS for the cluster", func() {
		if testContext.provider == "vagrant" {
			By("Skipping test which is broken for vagrant (See https://github.com/GoogleCloudPlatform/kubernetes/issues/3580)")
			return
		}

		podClient := c.Pods(api.NamespaceDefault)

		//TODO: Wait for skyDNS

		// All the names we need to be able to resolve.
		namesToResolve := []string{
			"kubernetes-ro",
			"kubernetes-ro.default",
			"kubernetes-ro.default.kubernetes.local",
			"google.com",
		}

		probeCmd := "for i in `seq 1 600`; do "
		for _, name := range namesToResolve {
			probeCmd += fmt.Sprintf("wget -O /dev/null %s && echo OK > /results/%s;", name, name)
		}
		probeCmd += "sleep 1; done"

		// Run a pod which probes DNS and exposes the results by HTTP.
		By("creating a pod to probe DNS")
		pod := &api.Pod{
			TypeMeta: api.TypeMeta{
				Kind:       "Pod",
				APIVersion: "v1beta1",
			},
			ObjectMeta: api.ObjectMeta{
				Name: "dns-test-" + string(util.NewUUID()),
			},
			Spec: api.PodSpec{
				Volumes: []api.Volume{
					{
						Name: "results",
						Source: api.VolumeSource{
							EmptyDir: &api.EmptyDir{},
						},
					},
				},
				Containers: []api.Container{
					{
						Name:  "webserver",
						Image: "kubernetes/test-webserver",
						VolumeMounts: []api.VolumeMount{
							{
								Name:      "results",
								MountPath: "/results",
							},
						},
					},
					{
						Name:    "pinger",
						Image:   "busybox",
						Command: []string{"sh", "-c", probeCmd},
						VolumeMounts: []api.VolumeMount{
							{
								Name:      "results",
								MountPath: "/results",
							},
						},
					},
				},
			},
		}

		By("submitting the pod to kuberenetes")
		_, err := podClient.Create(pod)
		if err != nil {
			Fail(fmt.Sprintf("Failed to create %s pod: %v", pod.Name, err))
		}
		defer func() {
			By("deleting the pod")
			defer GinkgoRecover()
			podClient.Delete(pod.Name)
		}()

		By("waiting for the pod to start running")
		waitForPodRunning(c, pod.Name)

		By("retrieving the pod")
		pod, err = podClient.Get(pod.Name)
		if err != nil {
			Fail(fmt.Sprintf("Failed to get pod %s: %v", pod.Name, err))
		}

		// Try to find results for each expected name.
		By("looking for the results for each expected name")
		var failed []string
		for try := 1; try < 100; try++ {
			failed = []string{}
			for _, name := range namesToResolve {
				_, err := c.Get().
					Prefix("proxy").
					Resource("pods").
					Namespace("default").
					Name(pod.Name).
					Suffix("results", name).
					Do().Raw()
				if err != nil {
					failed = append(failed, name)
					fmt.Printf("Lookup using %s for %s failed: %v\n", pod.Name, name, err)
				}
			}
			if len(failed) == 0 {
				break
			}
			fmt.Printf("lookups using %s failed for: %v\n", pod.Name, failed)
			time.Sleep(10 * time.Second)
		}
		Expect(len(failed)).To(Equal(0))

		// TODO: probe from the host, too.

		fmt.Printf("DNS probes using %s succeeded\n", pod.Name)
	})

	It("should provide RW and RO services", func() {
		svc := api.ServiceList{}
		err := c.Get().
			Namespace("default").
			AbsPath("/api/v1beta1/proxy/services/kubernetes-ro/api/v1beta1/services").
			Do().
			Into(&svc)
		if err != nil {
			Fail(fmt.Sprintf("unexpected error listing services using ro service: %v", err))
		}
		var foundRW, foundRO bool
		for i := range svc.Items {
			if svc.Items[i].Name == "kubernetes" {
				foundRW = true
			}
			if svc.Items[i].Name == "kubernetes-ro" {
				foundRO = true
			}
		}
		Expect(foundRW).To(Equal(true))
		Expect(foundRO).To(Equal(true))
	})
})
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

// stage - utility for generating C code consisting of a sequence of function
// calls ordered by stage number.
//
// This package is used by sysinit and sysdown.

package stage

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"

	log "github.com/Sirupsen/logrus"

	"mynewt.apache.org/newt/newt/pkg"
	"mynewt.apache.org/newt/util"
)

type StageFunc struct {
	Stage      int
	Name       string
	ReturnType string
	ArgList    string
	Pkg        *pkg.LocalPackage
}

type stageFuncSorter struct {
	// Used in logging; either "sysinit" or "sysdown".
	funcType string
	// Array of functions to be sorted.
	fns []StageFunc
}

func (s stageFuncSorter) Len() int {
	return len(s.fns)
}

func (s stageFuncSorter) Swap(i, j int) {
	s.fns[i], s.fns[j] = s.fns[j], s.fns[i]
}

func (s stageFuncSorter) Less(i, j int) bool {
	a := s.fns[i]
	b := s.fns[j]

	// 1: Sort by stage number.
	if a.Stage < b.Stage {
		return true
	} else if b.Stage < a.Stage {
		return false
	}

	// 2: Sort by function name.
	switch strings.Compare(a.Name, b.Name) {
	case -1:
		return true
	case 1:
		return false
	}

	// Same stage and function name?
	log.Warnf("Warning: Identical %s functions detected: %s",
		s.funcType, a.Name)

	return true
}

// SortStageFuncs performs an in-place sort of the provided StageFunc slice.
func SortStageFuncs(unsorted []StageFunc, funcType string) {
	s := stageFuncSorter{
		funcType: funcType,
		fns:      unsorted,
	}

	sort.Sort(s)
}

func (f *StageFunc) ReturnTypeString() string {
	if f.ReturnType == "" {
		return "void"
	} else {
		return f.ReturnType
	}
}

func (f *StageFunc) ArgListString() string {
	if f.ArgList == "" {
		return "void"
	} else {
		return f.ArgList
	}
}

// WriteCalls emits C code: a list of function prototypes corresponding to the
// provided slice of stage functions.
func WritePrototypes(sortedFns []StageFunc, w io.Writer) {
	for _, f := range sortedFns {
		fmt.Fprintf(w, "%s %s(%s);\n",
			f.ReturnTypeString(), f.Name, f.ArgListString())
	}
}

// WriteCalls emits C code: a sequence of function calls corresponding to the
// provided slice of stage functions.
func WriteCalls(sortedFuncs []StageFunc, argList string, w io.Writer) {
	prevStage := -1
	dupCount := 0

	for i, f := range sortedFuncs {
		if f.Stage != prevStage {
			prevStage = f.Stage
			dupCount = 0

			if i != 0 {
				fmt.Fprintf(w, "\n")
			}
			fmt.Fprintf(w, "    /*** Stage %d */\n", f.Stage)
		} else {
			dupCount += 1
		}

		fmt.Fprintf(w, "    /* %d.%d: %s (%s) */\n",
			f.Stage, dupCount, f.Name, f.Pkg.Name())
		fmt.Fprintf(w, "    %s(%s);\n", f.Name, argList)
	}
}

// WriteArr emits C code: an array body of function pointers represented by the
// supplied slice.  The caller must 1) write the array declaration before
// calling this function, and 2) write "};" afterwards.
func WriteArr(sortedFuncs []StageFunc, w io.Writer) {
	prevStage := -1
	dupCount := 0

	for i, f := range sortedFuncs {
		if f.Stage != prevStage {
			prevStage = f.Stage
			dupCount = 0

			if i != 0 {
				fmt.Fprintf(w, "\n")
			}
			fmt.Fprintf(w, "    /*** Stage %d */\n", f.Stage)
		} else {
			dupCount += 1
		}

		fmt.Fprintf(w, "    /* %d.%d: %s (%s) */\n",
			f.Stage, dupCount, f.Name, f.Pkg.Name())
		fmt.Fprintf(w, "    %s,\n", f.Name)
	}
	fmt.Fprintf(w, "\n")
	fmt.Fprintf(w, "    /*** Array terminator. */\n")
	fmt.Fprintf(w, "    0\n")
}

// EnsureWritten writes the specified file if its contents differ from those of
// the supplied byte slice.
func EnsureWritten(path string, contents []byte) error {
	unchanged, err := util.FileContains(contents, path)
	if err != nil {
		return err
	}

	if unchanged {
		log.Debugf("file unchanged; not writing src file (%s).", path)
		return nil
	}

	log.Debugf("file changed; writing src file (%s).", path)

	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
		return util.NewNewtError(err.Error())
	}

	if err := ioutil.WriteFile(path, contents, 0644); err != nil {
		return util.NewNewtError(err.Error())
	}

	return nil
}
<commit_msg>stage: Use ValSetting (generic setting value)<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

// stage - utility for generating C code consisting of a sequence of function
// calls ordered by stage number.
//
// This package is used by sysinit and sysdown.

package stage

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"

	log "github.com/Sirupsen/logrus"

	"mynewt.apache.org/newt/newt/pkg"
	"mynewt.apache.org/newt/newt/syscfg"
	"mynewt.apache.org/newt/newt/val"
	"mynewt.apache.org/newt/util"
)

type StageFunc struct {
	Stage      val.ValSetting
	Name       string
	ReturnType string
	ArgList    string
	Pkg        *pkg.LocalPackage
}

func NewStageFunc(name string, textVal string,
	p *pkg.LocalPackage, cfg *syscfg.Cfg) (StageFunc, error) {

	vs, err := val.ResolveValSetting(textVal, cfg)
	if err != nil {
		return StageFunc{}, err
	}

	// Ensure setting resolves to an integer.
	if _, err := vs.IntVal(); err != nil {
		return StageFunc{}, util.FmtNewtError("Invalid stage setting: %s=%s; "+
			"value does not resolve to an integer",
			name, textVal)
	}

	sf := StageFunc{
		Name:  name,
		Stage: vs,
		Pkg:   p,
	}

	return sf, nil
}

type stageFuncSorter struct {
	// Used in logging; either "sysinit" or "sysdown".
	funcType string
	// Array of functions to be sorted.
	fns []StageFunc
}

func (s stageFuncSorter) Len() int {
	return len(s.fns)
}

func (s stageFuncSorter) Swap(i, j int) {
	s.fns[i], s.fns[j] = s.fns[j], s.fns[i]
}

func (s stageFuncSorter) Less(i, j int) bool {
	a := s.fns[i]
	b := s.fns[j]

	inta, _ := a.Stage.IntVal()
	intb, _ := b.Stage.IntVal()

	// 1: Sort by stage number.
	if inta < intb {
		return true
	} else if intb < inta {
		return false
	}

	// 2: Sort by function name.
	switch strings.Compare(a.Name, b.Name) {
	case -1:
		return true
	case 1:
		return false
	}

	// Same stage and function name?
	log.Warnf("Warning: Identical %s functions detected: %s",
		s.funcType, a.Name)

	return true
}

// SortStageFuncs performs an in-place sort of the provided StageFunc slice.
func SortStageFuncs(unsorted []StageFunc, funcType string) {
	s := stageFuncSorter{
		funcType: funcType,
		fns:      unsorted,
	}

	sort.Sort(s)
}

func (f *StageFunc) ReturnTypeString() string {
	if f.ReturnType == "" {
		return "void"
	} else {
		return f.ReturnType
	}
}

func (f *StageFunc) ArgListString() string {
	if f.ArgList == "" {
		return "void"
	} else {
		return f.ArgList
	}
}

// WriteCalls emits C code: a list of function prototypes corresponding to the
// provided slice of stage functions.
func WritePrototypes(sortedFns []StageFunc, w io.Writer) {
	for _, f := range sortedFns {
		fmt.Fprintf(w, "%s %s(%s);\n",
			f.ReturnTypeString(), f.Name, f.ArgListString())
	}
}

// WriteCalls emits C code: a sequence of function calls corresponding to the
// provided slice of stage functions.
func WriteCalls(sortedFuncs []StageFunc, argList string, w io.Writer) {
	prevStage := -1
	dupCount := 0

	for i, f := range sortedFuncs {
		intStage, _ := f.Stage.IntVal()

		if intStage != prevStage {
			prevStage = intStage
			dupCount = 0

			if i != 0 {
				fmt.Fprintf(w, "\n")
			}
			fmt.Fprintf(w, "    /*** Stage %d */\n", intStage)
		} else {
			dupCount += 1
		}

		fmt.Fprintf(w, "    /* %d.%d: %s (%s) */\n",
			intStage, dupCount, f.Name, f.Pkg.Name())
		fmt.Fprintf(w, "    %s(%s);\n", f.Name, argList)
	}
}

// WriteArr emits C code: an array body of function pointers represented by the
// supplied slice.  The caller must 1) write the array declaration before
// calling this function, and 2) write "};" afterwards.
func WriteArr(sortedFuncs []StageFunc, w io.Writer) {
	prevStage := -1
	dupCount := 0

	for i, f := range sortedFuncs {
		intStage, _ := f.Stage.IntVal()

		if intStage != prevStage {
			prevStage = intStage
			dupCount = 0

			if i != 0 {
				fmt.Fprintf(w, "\n")
			}
			fmt.Fprintf(w, "    /*** Stage %d */\n", intStage)
		} else {
			dupCount += 1
		}

		fmt.Fprintf(w, "    /* %d.%d: %s (%s) */\n",
			intStage, dupCount, f.Name, f.Pkg.Name())
		fmt.Fprintf(w, "    %s,\n", f.Name)
	}
	fmt.Fprintf(w, "\n")
	fmt.Fprintf(w, "    /*** Array terminator. */\n")
	fmt.Fprintf(w, "    0\n")
}

// EnsureWritten writes the specified file if its contents differ from those of
// the supplied byte slice.
func EnsureWritten(path string, contents []byte) error {
	unchanged, err := util.FileContains(contents, path)
	if err != nil {
		return err
	}

	if unchanged {
		log.Debugf("file unchanged; not writing src file (%s).", path)
		return nil
	}

	log.Debugf("file changed; writing src file (%s).", path)

	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
		return util.NewNewtError(err.Error())
	}

	if err := ioutil.WriteFile(path, contents, 0644); err != nil {
		return util.NewNewtError(err.Error())
	}

	return nil
}
<|endoftext|>


<commit_before>package events

import (
	"context"
	"fmt"

	"github.com/hashicorp/nomad/api"
	"github.com/hashicorp/nomad/e2e/e2eutil"
	"github.com/hashicorp/nomad/e2e/framework"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/testutil"
	"github.com/stretchr/testify/require"
)

type EventsTest struct {
	framework.TC
	jobIDs []string
}

func init() {
	framework.AddSuites(&framework.TestSuite{
		Component:   "Events",
		CanRunLocal: true,
		Cases: []framework.TestCase{
			new(EventsTest),
		},
	})
}

func (tc *EventsTest) BeforeAll(f *framework.F) {
	e2eutil.WaitForLeader(f.T(), tc.Nomad())
}

func (tc *EventsTest) AfterEach(f *framework.F) {
	nomadClient := tc.Nomad()
	j := nomadClient.Jobs()

	for _, id := range tc.jobIDs {
		j.Deregister(id, true, nil)
	}
	_, err := e2eutil.Command("nomad", "system", "gc")
	f.NoError(err)
}

// TestDeploymentEvents registers a job then applies a change
// An event stream listening to Deployment Events asserts that
// a DeploymentPromotion event is emitted
func (tc *EventsTest) TestDeploymentEvents(f *framework.F) {
	t := f.T()

	nomadClient := tc.Nomad()
	events := nomadClient.EventStream()

	uuid := uuid.Generate()
	jobID := fmt.Sprintf("deployment-%s", uuid[0:8])
	tc.jobIDs = append(tc.jobIDs, jobID)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	topics := map[api.Topic][]string{
		"Deployment": {jobID},
	}

	var deployEvents []api.Event
	streamCh, err := events.Stream(ctx, topics, 0, nil)
	require.NoError(t, err)

	// gather deployment events
	go func() {
		for {
			select {
			case <-ctx.Done():
				return
			case event := <-streamCh:
				if event.IsHeartbeat() {
					continue
				}

				deployEvents = append(deployEvents, event.Events...)
			}
		}
	}()

	// register job
	e2eutil.RegisterAndWaitForAllocs(t, nomadClient, "events/input/initial.nomad", jobID, "")

	// update job
	e2eutil.RegisterAllocs(t, nomadClient, "events/input/deploy.nomad", jobID, "")

	ds := e2eutil.DeploymentsForJob(t, nomadClient, jobID)
	require.Equal(t, 2, len(ds))
	deploy := ds[0]

	// wait for deployment to be running and ready for auto promote
	e2eutil.WaitForDeployment(t, nomadClient, deploy.ID, structs.DeploymentStatusRunning, structs.DeploymentStatusDescriptionRunningAutoPromotion)

	// ensure there is a deployment promotion event
	testutil.WaitForResult(func() (bool, error) {
		for _, e := range deployEvents {
			if e.Type == "DeploymentPromotion" {
				return true, nil
			}
		}
		var got []string
		for _, e := range deployEvents {
			got = append(got, e.Type)
		}
		return false, fmt.Errorf("expected to receive deployment promotion event, got: %#v", got)
	}, func(e error) {
		f.NoError(e)
	})
}

// TestBlockedEvalEvents applies a job with a large memory requirement. The
// event stream checks for a failed task group alloc
func (tc *EventsTest) TestBlockedEvalEvents(f *framework.F) {
	t := f.T()

	nomadClient := tc.Nomad()
	events := nomadClient.EventStream()

	uuid := uuid.Generate()
	jobID := fmt.Sprintf("blocked-deploy-%s", uuid[0:8])
	tc.jobIDs = append(tc.jobIDs, jobID)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	topics := map[api.Topic][]string{
		"Eval": {"*"},
	}

	var evalEvents []api.Event
	streamCh, err := events.Stream(ctx, topics, 0, nil)
	require.NoError(t, err)

	// gather deployment events
	go func() {
		for {
			select {
			case <-ctx.Done():
				return
			case event := <-streamCh:
				if event.IsHeartbeat() {
					continue
				}

				evalEvents = append(evalEvents, event.Events...)
			}
		}
	}()

	// register job
	e2eutil.Register(jobID, "events/input/large-job.nomad")

	// ensure there is a deployment promotion event
	testutil.WaitForResult(func() (bool, error) {
		for _, e := range evalEvents {
			evalRaw, ok := e.Payload["Eval"].(map[string]interface{})
			if !ok {
				return false, fmt.Errorf("type assertion on eval")
			}

			ftg, ok := evalRaw["FailedTGAllocs"].(map[string]interface{})
			if !ok {
				continue
			}

			tg, ok := ftg["one"].(map[string]interface{})
			if !ok {
				continue
			}
			mem := tg["DimensionExhausted"].(map[string]interface{})["memory"]
			require.NotNil(t, mem, "memory dimension was nil")
			memInt := int(mem.(float64))
			require.Greater(t, memInt, 0, "memory dimension was zero")
			return true, nil

		}
		return false, fmt.Errorf("expected blocked eval with memory exhausted, got: %#v", evalEvents)
	}, func(e error) {
		require.NoError(t, e)
	})
}
<commit_msg>events: e2e test that API client honors the index flag<commit_after>package events

import (
	"context"
	"fmt"

	"github.com/hashicorp/nomad/api"
	"github.com/hashicorp/nomad/e2e/e2eutil"
	"github.com/hashicorp/nomad/e2e/framework"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/testutil"
	"github.com/stretchr/testify/require"
)

type EventsTest struct {
	framework.TC
	jobIDs []string
}

func init() {
	framework.AddSuites(&framework.TestSuite{
		Component:   "Events",
		CanRunLocal: true,
		Cases: []framework.TestCase{
			new(EventsTest),
		},
	})
}

func (tc *EventsTest) BeforeAll(f *framework.F) {
	e2eutil.WaitForLeader(f.T(), tc.Nomad())
}

func (tc *EventsTest) AfterEach(f *framework.F) {
	nomadClient := tc.Nomad()
	j := nomadClient.Jobs()

	for _, id := range tc.jobIDs {
		j.Deregister(id, true, nil)
	}
	_, err := e2eutil.Command("nomad", "system", "gc")
	f.NoError(err)
}

// TestDeploymentEvents registers a job then applies a change
// An event stream listening to Deployment Events asserts that
// a DeploymentPromotion event is emitted
func (tc *EventsTest) TestDeploymentEvents(f *framework.F) {
	t := f.T()

	nomadClient := tc.Nomad()
	events := nomadClient.EventStream()

	uuid := uuid.Generate()
	jobID := fmt.Sprintf("deployment-%s", uuid[0:8])
	tc.jobIDs = append(tc.jobIDs, jobID)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	topics := map[api.Topic][]string{
		"Deployment": {jobID},
	}

	var deployEvents []api.Event
	streamCh, err := events.Stream(ctx, topics, 0, nil)
	require.NoError(t, err)

	// gather deployment events
	go func() {
		for {
			select {
			case <-ctx.Done():
				return
			case event := <-streamCh:
				if event.IsHeartbeat() {
					continue
				}

				deployEvents = append(deployEvents, event.Events...)
			}
		}
	}()

	// register job
	e2eutil.RegisterAndWaitForAllocs(t, nomadClient, "events/input/initial.nomad", jobID, "")

	// update job
	e2eutil.RegisterAllocs(t, nomadClient, "events/input/deploy.nomad", jobID, "")

	ds := e2eutil.DeploymentsForJob(t, nomadClient, jobID)
	require.Equal(t, 2, len(ds))
	deploy := ds[0]

	// wait for deployment to be running and ready for auto promote
	e2eutil.WaitForDeployment(t, nomadClient, deploy.ID, structs.DeploymentStatusRunning, structs.DeploymentStatusDescriptionRunningAutoPromotion)

	// ensure there is a deployment promotion event
	testutil.WaitForResult(func() (bool, error) {
		for _, e := range deployEvents {
			if e.Type == "DeploymentPromotion" {
				return true, nil
			}
		}
		var got []string
		for _, e := range deployEvents {
			got = append(got, e.Type)
		}
		return false, fmt.Errorf("expected to receive deployment promotion event, got: %#v", got)
	}, func(e error) {
		f.NoError(e)
	})
}

// TestBlockedEvalEvents applies a job with a large memory requirement. The
// event stream checks for a failed task group alloc
func (tc *EventsTest) TestBlockedEvalEvents(f *framework.F) {
	t := f.T()

	nomadClient := tc.Nomad()
	events := nomadClient.EventStream()

	uuid := uuid.Generate()
	jobID := fmt.Sprintf("blocked-deploy-%s", uuid[0:8])
	tc.jobIDs = append(tc.jobIDs, jobID)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	topics := map[api.Topic][]string{
		"Eval": {"*"},
	}

	var evalEvents []api.Event
	streamCh, err := events.Stream(ctx, topics, 0, nil)
	require.NoError(t, err)

	// gather deployment events
	go func() {
		for {
			select {
			case <-ctx.Done():
				return
			case event := <-streamCh:
				if event.IsHeartbeat() {
					continue
				}

				evalEvents = append(evalEvents, event.Events...)
			}
		}
	}()

	// register job
	e2eutil.Register(jobID, "events/input/large-job.nomad")

	// ensure there is a deployment promotion event
	testutil.WaitForResult(func() (bool, error) {
		for _, e := range evalEvents {
			evalRaw, ok := e.Payload["Eval"].(map[string]interface{})
			if !ok {
				return false, fmt.Errorf("type assertion on eval")
			}

			ftg, ok := evalRaw["FailedTGAllocs"].(map[string]interface{})
			if !ok {
				continue
			}

			tg, ok := ftg["one"].(map[string]interface{})
			if !ok {
				continue
			}
			mem := tg["DimensionExhausted"].(map[string]interface{})["memory"]
			require.NotNil(t, mem, "memory dimension was nil")
			memInt := int(mem.(float64))
			require.Greater(t, memInt, 0, "memory dimension was zero")
			return true, nil

		}
		return false, fmt.Errorf("expected blocked eval with memory exhausted, got: %#v", evalEvents)
	}, func(e error) {
		require.NoError(t, e)
	})
}

// TestStartIndex applies a job, then connects to the stream with a start
// index to verify that the events from before the job are not included.
func (tc *EventsTest) TestStartIndex(f *framework.F) {
	t := f.T()

	nomadClient := tc.Nomad()
	events := nomadClient.EventStream()

	uuid := uuid.Generate()
	jobID := fmt.Sprintf("deployment-%s", uuid[0:8])
	jobID2 := fmt.Sprintf("deployment2-%s", uuid[0:8])
	tc.jobIDs = append(tc.jobIDs, jobID, jobID2)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// register job
	err := e2eutil.Register(jobID, "events/input/initial.nomad")
	require.NoError(t, err)
	job, _, err := nomadClient.Jobs().Info(jobID, nil)
	require.NoError(t, err)
	startIndex := *job.JobModifyIndex + 1

	topics := map[api.Topic][]string{
		"Job": {"*"},
	}

	// starting at Job.ModifyIndex + 1, the next (and only) JobRegistered event that we see
	// should be from a different job registration
	streamCh, err := events.Stream(ctx, topics, startIndex, nil)
	require.NoError(t, err)

	var jobEvents []api.Event
	// gather job register events
	go func() {
		for {
			select {
			case <-ctx.Done():
				return
			case event := <-streamCh:
				if event.IsHeartbeat() {
					continue
				}
				jobEvents = append(jobEvents, event.Events...)
			}
		}
	}()

	// new job (to make sure we get a JobRegistered event)
	err = e2eutil.Register(jobID2, "events/input/deploy.nomad")
	require.NoError(t, err)

	// ensure there is a deployment promotion event
	foundUnexpected := false
	testutil.WaitForResult(func() (bool, error) {
		for _, e := range jobEvents {
			if e.Type == "JobRegistered" {
				if e.Index <= startIndex {
					foundUnexpected = true
				}
				if e.Index >= startIndex {
					return true, nil
				}
			}
		}
		return false, fmt.Errorf("expected to receive JobRegistered event for index at least %v", startIndex)
	}, func(e error) {
		f.NoError(e)
	})
	require.False(t, foundUnexpected, "found events from earlier-than-expected indices")
}
<|endoftext|>