<commit_before>package herd

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/srpc"
	subproto "github.com/Symantec/Dominator/proto/sub"
	"github.com/Symantec/Dominator/sub/client"
	"net"
	"runtime"
	"strings"
	"time"
)

var (
	subConnectTimeout = flag.Uint("subConnectTimeout", 15,
		"Timeout in seconds for sub connections. If zero, OS timeout is used")
	logUnknownSubConnectErrors = flag.Bool("logUnknownSubConnectErrors", false,
		"If true, log unknown sub connection errors")
)

func (sub *Sub) tryMakeBusy() bool {
	sub.busyMutex.Lock()
	defer sub.busyMutex.Unlock()
	if sub.busy {
		return false
	}
	sub.busyStartTime = time.Now()
	sub.busy = true
	return true
}

func (sub *Sub) makeUnbusy() {
	sub.busyMutex.Lock()
	defer sub.busyMutex.Unlock()
	sub.busyStopTime = time.Now()
	sub.busy = false
}

func (sub *Sub) connectAndPoll() {
	previousStatus := sub.status
	sub.status = statusConnecting
	hostname := strings.SplitN(sub.hostname, "*", 2)[0]
	address := fmt.Sprintf("%s:%d", hostname, constants.SubPortNumber)
	sub.lastConnectionStartTime = time.Now()
	srpcClient, err := srpc.DialHTTP("tcp", address,
		time.Second*time.Duration(*subConnectTimeout))
	dialReturnedTime := time.Now()
	if err != nil {
		sub.pollTime = time.Time{}
		if err, ok := err.(*net.OpError); ok {
			if _, ok := err.Err.(*net.DNSError); ok {
				sub.status = statusDNSError
				return
			}
			if err.Timeout() {
				sub.status = statusConnectTimeout
				return
			}
		}
		if err == srpc.ErrorConnectionRefused {
			sub.status = statusConnectionRefused
			return
		}
		if err == srpc.ErrorNoRouteToHost {
			sub.status = statusNoRouteToHost
			return
		}
		if err == srpc.ErrorMissingCertificate {
			sub.lastReachableTime = dialReturnedTime
			sub.status = statusMissingCertificate
			return
		}
		if err == srpc.ErrorBadCertificate {
			sub.lastReachableTime = dialReturnedTime
			sub.status = statusBadCertificate
			return
		}
		sub.status = statusFailedToConnect
		if *logUnknownSubConnectErrors {
			sub.herd.logger.Println(err)
		}
		return
	}
	defer srpcClient.Close()
	sub.status = statusWaitingToPoll
	sub.lastReachableTime = dialReturnedTime
	sub.lastConnectionSucceededTime = dialReturnedTime
	sub.lastConnectDuration =
		sub.lastConnectionSucceededTime.Sub(sub.lastConnectionStartTime)
	connectDistribution.Add(sub.lastConnectDuration)
	sub.herd.pollSemaphore <- true
	sub.status = statusPolling
	sub.poll(srpcClient, previousStatus)
	<-sub.herd.pollSemaphore
}

func (sub *Sub) poll(srpcClient *srpc.Client, previousStatus subStatus) {
	// If the planned image has just become available, force a full poll.
	if previousStatus == statusSynced &&
		!sub.havePlannedImage &&
		sub.herd.getImageNoError(sub.plannedImage) != nil {
		sub.havePlannedImage = true
		sub.generationCount = 0 // Force a full poll.
	}
	var request subproto.PollRequest
	request.HaveGeneration = sub.generationCount
	var reply subproto.PollResponse
	sub.lastPollStartTime = time.Now()
	haveImage := false
	if sub.herd.getImageNoError(sub.requiredImage) == nil {
		request.ShortPollOnly = true
	} else {
		haveImage = true
	}
	logger := sub.herd.logger
	if err := client.CallPoll(srpcClient, request, &reply); err != nil {
		sub.pollTime = time.Time{}
		if err == srpc.ErrorAccessToMethodDenied {
			sub.status = statusPollDenied
		} else {
			sub.status = statusFailedToPoll
		}
		logger.Printf("Error calling %s.Poll()\t%s\n", sub.hostname, err)
		return
	}
	sub.lastPollSucceededTime = time.Now()
	if reply.GenerationCount == 0 {
		sub.reclaim()
		sub.generationCount = 0
	}
	if fs := reply.FileSystem; fs == nil {
		sub.lastShortPollDuration =
			sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
		shortPollDistribution.Add(sub.lastShortPollDuration)
		if sub.startTime != reply.StartTime {
			sub.generationCount = 0 // Sub has restarted: force a full poll.
		}
	} else {
		if err := fs.RebuildInodePointers(); err != nil {
			sub.status = statusFailedToPoll
			logger.Printf("Error building pointers for: %s %s\n",
				sub.hostname, err)
			return
		}
		fs.BuildEntryMap()
		sub.fileSystem = fs
		sub.objectCache = reply.ObjectCache
		sub.generationCount = reply.GenerationCount
		sub.lastFullPollDuration =
			sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
		fullPollDistribution.Add(sub.lastFullPollDuration)
		logger.Printf("Polled: %s, GenerationCount=%d\n",
			sub.hostname, reply.GenerationCount)
	}
	sub.startTime = reply.StartTime
	sub.pollTime = reply.PollTime
	if reply.FetchInProgress {
		sub.status = statusFetching
		return
	}
	if reply.UpdateInProgress {
		sub.status = statusUpdating
		return
	}
	if reply.GenerationCount < 1 {
		sub.status = statusSubNotReady
		return
	}
	if previousStatus == statusFetching && reply.LastFetchError != "" {
		logger.Printf("Fetch failure for: %s: %s\n",
			sub.hostname, reply.LastFetchError)
		sub.status = statusFailedToFetch
		if sub.fileSystem == nil {
			sub.generationCount = 0 // Force a full poll next cycle.
			return
		}
	}
	if previousStatus == statusUpdating {
		// Transition from updating to update ended (may be partial/failed).
		if reply.LastUpdateError != "" {
			logger.Printf("Update failure for: %s: %s\n",
				sub.hostname, reply.LastUpdateError)
			sub.status = statusFailedToUpdate
		} else {
			sub.status = statusWaitingForNextFullPoll
		}
		sub.scanCountAtLastUpdateEnd = reply.ScanCount
		sub.reclaim()
		return
	}
	if !haveImage {
		sub.status = statusImageNotReady
		return
	}
	if previousStatus == statusFailedToUpdate ||
		previousStatus == statusWaitingForNextFullPoll {
		if sub.scanCountAtLastUpdateEnd == reply.ScanCount {
			// Need to wait until sub has performed a new scan.
			if sub.fileSystem != nil {
				sub.reclaim()
			}
			sub.status = previousStatus
			return
		}
		if sub.fileSystem == nil {
			// Force a full poll next cycle so that we can see the state of the
			// sub.
			sub.generationCount = 0
			sub.status = previousStatus
			return
		}
	}
	if sub.fileSystem == nil {
		sub.status = previousStatus
		return
	}
	if idle, status := sub.fetchMissingObjects(srpcClient,
		sub.requiredImage); !idle {
		sub.status = status
		sub.reclaim()
		return
	}
	sub.status = statusComputingUpdate
	if idle, status := sub.sendUpdate(srpcClient); !idle {
		sub.status = status
		sub.reclaim()
		return
	}
	if idle, status := sub.fetchMissingObjects(srpcClient,
		sub.plannedImage); !idle {
		if status != statusImageNotReady {
			sub.status = status
			sub.reclaim()
			return
		}
	}
	if previousStatus == statusWaitingForNextFullPoll &&
		!sub.lastUpdateTime.IsZero() {
		sub.lastSyncTime = time.Now()
	}
	sub.status = statusSynced
	sub.cleanup(srpcClient, sub.plannedImage)
	sub.reclaim()
}

func (sub *Sub) reclaim() {
	sub.fileSystem = nil  // Mark memory for reclaim.
	sub.objectCache = nil // Mark memory for reclaim.
	runtime.GC()          // Reclaim now.
}

// Returns true if all required objects are available.
func (sub *Sub) fetchMissingObjects(srpcClient *srpc.Client, imageName string) (
	bool, subStatus) {
	image := sub.herd.getImageNoError(imageName)
	if image == nil {
		return false, statusImageNotReady
	}
	missingObjects := make(map[hash.Hash]struct{})
	for _, inode := range image.FileSystem.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				missingObjects[inode.Hash] = struct{}{}
			}
		}
	}
	for _, hash := range sub.objectCache {
		delete(missingObjects, hash)
	}
	for _, inode := range sub.fileSystem.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				delete(missingObjects, inode.Hash)
			}
		}
	}
	if len(missingObjects) < 1 {
		return true, statusSynced
	}
	logger := sub.herd.logger
	logger.Printf("Calling %s.Fetch() for: %d objects\n",
		sub.hostname, len(missingObjects))
	var request subproto.FetchRequest
	var reply subproto.FetchResponse
	request.ServerAddress = sub.herd.imageServerAddress
	for hash := range missingObjects {
		request.Hashes = append(request.Hashes, hash)
	}
	if err := client.CallFetch(srpcClient, request, &reply); err != nil {
		logger.Printf("Error calling %s.Fetch()\t%s\n", sub.hostname, err)
		if err == srpc.ErrorAccessToMethodDenied {
			return false, statusFetchDenied
		}
		return false, statusFailedToFetch
	}
	return false, statusFetching
}

// Returns true if no update needs to be performed.
func (sub *Sub) sendUpdate(srpcClient *srpc.Client) (bool, subStatus) {
	logger := sub.herd.logger
	var request subproto.UpdateRequest
	var reply subproto.UpdateResponse
	if sub.buildUpdateRequest(&request) {
		return true, statusSynced
	}
	sub.status = statusSendingUpdate
	sub.lastUpdateTime = time.Now()
	if err := client.CallUpdate(srpcClient, request, &reply); err != nil {
		logger.Printf("Error calling %s:Subd.Update()\t%s\n", sub.hostname, err)
		if err == srpc.ErrorAccessToMethodDenied {
			return false, statusUpdateDenied
		}
		return false, statusFailedToUpdate
	}
	return false, statusUpdating
}

func (sub *Sub) cleanup(srpcClient *srpc.Client, plannedImageName string) {
	logger := sub.herd.logger
	unusedObjects := make(map[hash.Hash]bool)
	for _, hash := range sub.objectCache {
		unusedObjects[hash] = false // Potential cleanup candidate.
	}
	for _, inode := range sub.fileSystem.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				if _, ok := unusedObjects[inode.Hash]; ok {
					unusedObjects[inode.Hash] = true // Must clean this one up.
				}
			}
		}
	}
	image := sub.herd.getImageNoError(plannedImageName)
	if image != nil {
		for _, inode := range image.FileSystem.InodeTable {
			if inode, ok := inode.(*filesystem.RegularInode); ok {
				if inode.Size > 0 {
					if clean, ok := unusedObjects[inode.Hash]; !clean && ok {
						delete(unusedObjects, inode.Hash)
					}
				}
			}
		}
	}
	if len(unusedObjects) < 1 {
		return
	}
	var request subproto.CleanupRequest
	var reply subproto.CleanupResponse
	request.Hashes = make([]hash.Hash, 0, len(unusedObjects))
	for hash := range unusedObjects {
		request.Hashes = append(request.Hashes, hash)
	}
	if err := client.CallCleanup(srpcClient, request, &reply); err != nil {
		logger.Printf("Error calling %s:Subd.Cleanup()\t%s\n",
			sub.hostname, err)
	}
}
<commit_msg>Fix time comparison in dominator.<commit_after>package herd

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/srpc"
	subproto "github.com/Symantec/Dominator/proto/sub"
	"github.com/Symantec/Dominator/sub/client"
	"net"
	"runtime"
	"strings"
	"time"
)

var (
	subConnectTimeout = flag.Uint("subConnectTimeout", 15,
		"Timeout in seconds for sub connections. If zero, OS timeout is used")
	logUnknownSubConnectErrors = flag.Bool("logUnknownSubConnectErrors", false,
		"If true, log unknown sub connection errors")
)

func (sub *Sub) tryMakeBusy() bool {
	sub.busyMutex.Lock()
	defer sub.busyMutex.Unlock()
	if sub.busy {
		return false
	}
	sub.busyStartTime = time.Now()
	sub.busy = true
	return true
}

func (sub *Sub) makeUnbusy() {
	sub.busyMutex.Lock()
	defer sub.busyMutex.Unlock()
	sub.busyStopTime = time.Now()
	sub.busy = false
}

func (sub *Sub) connectAndPoll() {
	previousStatus := sub.status
	sub.status = statusConnecting
	hostname := strings.SplitN(sub.hostname, "*", 2)[0]
	address := fmt.Sprintf("%s:%d", hostname, constants.SubPortNumber)
	sub.lastConnectionStartTime = time.Now()
	srpcClient, err := srpc.DialHTTP("tcp", address,
		time.Second*time.Duration(*subConnectTimeout))
	dialReturnedTime := time.Now()
	if err != nil {
		sub.pollTime = time.Time{}
		if err, ok := err.(*net.OpError); ok {
			if _, ok := err.Err.(*net.DNSError); ok {
				sub.status = statusDNSError
				return
			}
			if err.Timeout() {
				sub.status = statusConnectTimeout
				return
			}
		}
		if err == srpc.ErrorConnectionRefused {
			sub.status = statusConnectionRefused
			return
		}
		if err == srpc.ErrorNoRouteToHost {
			sub.status = statusNoRouteToHost
			return
		}
		if err == srpc.ErrorMissingCertificate {
			sub.lastReachableTime = dialReturnedTime
			sub.status = statusMissingCertificate
			return
		}
		if err == srpc.ErrorBadCertificate {
			sub.lastReachableTime = dialReturnedTime
			sub.status = statusBadCertificate
			return
		}
		sub.status = statusFailedToConnect
		if *logUnknownSubConnectErrors {
			sub.herd.logger.Println(err)
		}
		return
	}
	defer srpcClient.Close()
	sub.status = statusWaitingToPoll
	sub.lastReachableTime = dialReturnedTime
	sub.lastConnectionSucceededTime = dialReturnedTime
	sub.lastConnectDuration =
		sub.lastConnectionSucceededTime.Sub(sub.lastConnectionStartTime)
	connectDistribution.Add(sub.lastConnectDuration)
	sub.herd.pollSemaphore <- true
	sub.status = statusPolling
	sub.poll(srpcClient, previousStatus)
	<-sub.herd.pollSemaphore
}

func (sub *Sub) poll(srpcClient *srpc.Client, previousStatus subStatus) {
	// If the planned image has just become available, force a full poll.
	if previousStatus == statusSynced &&
		!sub.havePlannedImage &&
		sub.herd.getImageNoError(sub.plannedImage) != nil {
		sub.havePlannedImage = true
		sub.generationCount = 0 // Force a full poll.
	}
	var request subproto.PollRequest
	request.HaveGeneration = sub.generationCount
	var reply subproto.PollResponse
	sub.lastPollStartTime = time.Now()
	haveImage := false
	if sub.herd.getImageNoError(sub.requiredImage) == nil {
		request.ShortPollOnly = true
	} else {
		haveImage = true
	}
	logger := sub.herd.logger
	if err := client.CallPoll(srpcClient, request, &reply); err != nil {
		sub.pollTime = time.Time{}
		if err == srpc.ErrorAccessToMethodDenied {
			sub.status = statusPollDenied
		} else {
			sub.status = statusFailedToPoll
		}
		logger.Printf("Error calling %s.Poll()\t%s\n", sub.hostname, err)
		return
	}
	sub.lastPollSucceededTime = time.Now()
	if reply.GenerationCount == 0 {
		sub.reclaim()
		sub.generationCount = 0
	}
	if fs := reply.FileSystem; fs == nil {
		sub.lastShortPollDuration =
			sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
		shortPollDistribution.Add(sub.lastShortPollDuration)
		if !sub.startTime.Equal(reply.StartTime) {
			sub.generationCount = 0 // Sub has restarted: force a full poll.
		}
	} else {
		if err := fs.RebuildInodePointers(); err != nil {
			sub.status = statusFailedToPoll
			logger.Printf("Error building pointers for: %s %s\n",
				sub.hostname, err)
			return
		}
		fs.BuildEntryMap()
		sub.fileSystem = fs
		sub.objectCache = reply.ObjectCache
		sub.generationCount = reply.GenerationCount
		sub.lastFullPollDuration =
			sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
		fullPollDistribution.Add(sub.lastFullPollDuration)
		logger.Printf("Polled: %s, GenerationCount=%d\n",
			sub.hostname, reply.GenerationCount)
	}
	sub.startTime = reply.StartTime
	sub.pollTime = reply.PollTime
	if reply.FetchInProgress {
		sub.status = statusFetching
		return
	}
	if reply.UpdateInProgress {
		sub.status = statusUpdating
		return
	}
	if reply.GenerationCount < 1 {
		sub.status = statusSubNotReady
		return
	}
	if previousStatus == statusFetching && reply.LastFetchError != "" {
		logger.Printf("Fetch failure for: %s: %s\n",
			sub.hostname, reply.LastFetchError)
		sub.status = statusFailedToFetch
		if sub.fileSystem == nil {
			sub.generationCount = 0 // Force a full poll next cycle.
			return
		}
	}
	if previousStatus == statusUpdating {
		// Transition from updating to update ended (may be partial/failed).
		if reply.LastUpdateError != "" {
			logger.Printf("Update failure for: %s: %s\n",
				sub.hostname, reply.LastUpdateError)
			sub.status = statusFailedToUpdate
		} else {
			sub.status = statusWaitingForNextFullPoll
		}
		sub.scanCountAtLastUpdateEnd = reply.ScanCount
		sub.reclaim()
		return
	}
	if !haveImage {
		sub.status = statusImageNotReady
		return
	}
	if previousStatus == statusFailedToUpdate ||
		previousStatus == statusWaitingForNextFullPoll {
		if sub.scanCountAtLastUpdateEnd == reply.ScanCount {
			// Need to wait until sub has performed a new scan.
			if sub.fileSystem != nil {
				sub.reclaim()
			}
			sub.status = previousStatus
			return
		}
		if sub.fileSystem == nil {
			// Force a full poll next cycle so that we can see the state of the
			// sub.
			sub.generationCount = 0
			sub.status = previousStatus
			return
		}
	}
	if sub.fileSystem == nil {
		sub.status = previousStatus
		return
	}
	if idle, status := sub.fetchMissingObjects(srpcClient,
		sub.requiredImage); !idle {
		sub.status = status
		sub.reclaim()
		return
	}
	sub.status = statusComputingUpdate
	if idle, status := sub.sendUpdate(srpcClient); !idle {
		sub.status = status
		sub.reclaim()
		return
	}
	if idle, status := sub.fetchMissingObjects(srpcClient,
		sub.plannedImage); !idle {
		if status != statusImageNotReady {
			sub.status = status
			sub.reclaim()
			return
		}
	}
	if previousStatus == statusWaitingForNextFullPoll &&
		!sub.lastUpdateTime.IsZero() {
		sub.lastSyncTime = time.Now()
	}
	sub.status = statusSynced
	sub.cleanup(srpcClient, sub.plannedImage)
	sub.reclaim()
}

func (sub *Sub) reclaim() {
	sub.fileSystem = nil  // Mark memory for reclaim.
	sub.objectCache = nil // Mark memory for reclaim.
	runtime.GC()          // Reclaim now.
}

// Returns true if all required objects are available.
func (sub *Sub) fetchMissingObjects(srpcClient *srpc.Client, imageName string) (
	bool, subStatus) {
	image := sub.herd.getImageNoError(imageName)
	if image == nil {
		return false, statusImageNotReady
	}
	missingObjects := make(map[hash.Hash]struct{})
	for _, inode := range image.FileSystem.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				missingObjects[inode.Hash] = struct{}{}
			}
		}
	}
	for _, hash := range sub.objectCache {
		delete(missingObjects, hash)
	}
	for _, inode := range sub.fileSystem.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				delete(missingObjects, inode.Hash)
			}
		}
	}
	if len(missingObjects) < 1 {
		return true, statusSynced
	}
	logger := sub.herd.logger
	logger.Printf("Calling %s.Fetch() for: %d objects\n",
		sub.hostname, len(missingObjects))
	var request subproto.FetchRequest
	var reply subproto.FetchResponse
	request.ServerAddress = sub.herd.imageServerAddress
	for hash := range missingObjects {
		request.Hashes = append(request.Hashes, hash)
	}
	if err := client.CallFetch(srpcClient, request, &reply); err != nil {
		logger.Printf("Error calling %s.Fetch()\t%s\n", sub.hostname, err)
		if err == srpc.ErrorAccessToMethodDenied {
			return false, statusFetchDenied
		}
		return false, statusFailedToFetch
	}
	return false, statusFetching
}

// Returns true if no update needs to be performed.
func (sub *Sub) sendUpdate(srpcClient *srpc.Client) (bool, subStatus) {
	logger := sub.herd.logger
	var request subproto.UpdateRequest
	var reply subproto.UpdateResponse
	if sub.buildUpdateRequest(&request) {
		return true, statusSynced
	}
	sub.status = statusSendingUpdate
	sub.lastUpdateTime = time.Now()
	if err := client.CallUpdate(srpcClient, request, &reply); err != nil {
		logger.Printf("Error calling %s:Subd.Update()\t%s\n", sub.hostname, err)
		if err == srpc.ErrorAccessToMethodDenied {
			return false, statusUpdateDenied
		}
		return false, statusFailedToUpdate
	}
	return false, statusUpdating
}

func (sub *Sub) cleanup(srpcClient *srpc.Client, plannedImageName string) {
	logger := sub.herd.logger
	unusedObjects := make(map[hash.Hash]bool)
	for _, hash := range sub.objectCache {
		unusedObjects[hash] = false // Potential cleanup candidate.
	}
	for _, inode := range sub.fileSystem.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				if _, ok := unusedObjects[inode.Hash]; ok {
					unusedObjects[inode.Hash] = true // Must clean this one up.
				}
			}
		}
	}
	image := sub.herd.getImageNoError(plannedImageName)
	if image != nil {
		for _, inode := range image.FileSystem.InodeTable {
			if inode, ok := inode.(*filesystem.RegularInode); ok {
				if inode.Size > 0 {
					if clean, ok := unusedObjects[inode.Hash]; !clean && ok {
						delete(unusedObjects, inode.Hash)
					}
				}
			}
		}
	}
	if len(unusedObjects) < 1 {
		return
	}
	var request subproto.CleanupRequest
	var reply subproto.CleanupResponse
	request.Hashes = make([]hash.Hash, 0, len(unusedObjects))
	for hash := range unusedObjects {
		request.Hashes = append(request.Hashes, hash)
	}
	if err := client.CallCleanup(srpcClient, request, &reply); err != nil {
		logger.Printf("Error calling %s:Subd.Cleanup()\t%s\n",
			sub.hostname, err)
	}
}
<|endoftext|>


<commit_before>package herd

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/srpc"
	subproto "github.com/Symantec/Dominator/proto/sub"
	"github.com/Symantec/Dominator/sub/client"
	"net"
	"runtime"
	"strings"
	"time"
)

var (
	subConnectTimeout = flag.Uint("subConnectTimeout", 15,
		"Timeout in seconds for sub connections. If zero, OS timeout is used")
	logUnknownSubConnectErrors = flag.Bool("logUnknownSubConnectErrors", false,
		"If true, log unknown sub connection errors")
)

func (sub *Sub) tryMakeBusy() bool {
	sub.busyMutex.Lock()
	defer sub.busyMutex.Unlock()
	if sub.busy {
		return false
	}
	sub.busyStartTime = time.Now()
	sub.busy = true
	return true
}

func (sub *Sub) makeUnbusy() {
	sub.busyMutex.Lock()
	defer sub.busyMutex.Unlock()
	sub.busyStopTime = time.Now()
	sub.busy = false
}

func (sub *Sub) connectAndPoll() {
	previousStatus := sub.status
	sub.status = statusConnecting
	hostname := strings.SplitN(sub.hostname, "*", 2)[0]
	address := fmt.Sprintf("%s:%d", hostname, constants.SubPortNumber)
	sub.lastConnectionStartTime = time.Now()
	srpcClient, err := srpc.DialHTTP("tcp", address,
		time.Second*time.Duration(*subConnectTimeout))
	dialReturnedTime := time.Now()
	if err != nil {
		sub.pollTime = time.Time{}
		if err, ok := err.(*net.OpError); ok {
			if _, ok := err.Err.(*net.DNSError); ok {
				sub.status = statusDNSError
				return
			}
			if err.Timeout() {
				sub.status = statusConnectTimeout
				return
			}
		}
		if err == srpc.ErrorConnectionRefused {
			sub.status = statusConnectionRefused
			return
		}
		if err == srpc.ErrorNoRouteToHost {
			sub.status = statusNoRouteToHost
			return
		}
		if err == srpc.ErrorMissingCertificate {
			sub.lastReachableTime = dialReturnedTime
			sub.status = statusMissingCertificate
			return
		}
		if err == srpc.ErrorBadCertificate {
			sub.lastReachableTime = dialReturnedTime
			sub.status = statusBadCertificate
			return
		}
		sub.status = statusFailedToConnect
		if *logUnknownSubConnectErrors {
			sub.herd.logger.Println(err)
		}
		return
	}
	defer srpcClient.Close()
	sub.status = statusWaitingToPoll
	sub.lastReachableTime = dialReturnedTime
	sub.lastConnectionSucceededTime = dialReturnedTime
	sub.lastConnectDuration =
		sub.lastConnectionSucceededTime.Sub(sub.lastConnectionStartTime)
	connectDistribution.Add(sub.lastConnectDuration)
	sub.herd.pollSemaphore <- true
	sub.status = statusPolling
	sub.poll(srpcClient, previousStatus)
	<-sub.herd.pollSemaphore
}

func (sub *Sub) poll(srpcClient *srpc.Client, previousStatus subStatus) {
	// If the planned image has just become available, force a full poll.
	if previousStatus == statusSynced &&
		!sub.havePlannedImage &&
		sub.herd.getImageNoError(sub.plannedImage) != nil {
		sub.havePlannedImage = true
		sub.generationCount = 0 // Force a full poll.
	}
	var request subproto.PollRequest
	request.HaveGeneration = sub.generationCount
	var reply subproto.PollResponse
	sub.lastPollStartTime = time.Now()
	haveImage := false
	if sub.herd.getImageNoError(sub.requiredImage) == nil {
		request.ShortPollOnly = true
	} else {
		haveImage = true
	}
	logger := sub.herd.logger
	if err := client.CallPoll(srpcClient, request, &reply); err != nil {
		sub.pollTime = time.Time{}
		if err == srpc.ErrorAccessToMethodDenied {
			sub.status = statusPollDenied
		} else {
			sub.status = statusFailedToPoll
		}
		logger.Printf("Error calling %s.Poll()\t%s\n", sub.hostname, err)
		return
	}
	sub.lastPollSucceededTime = time.Now()
	if reply.GenerationCount == 0 {
		sub.reclaim()
		sub.generationCount = 0
	}
	if fs := reply.FileSystem; fs == nil {
		sub.lastShortPollDuration =
			sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
		shortPollDistribution.Add(sub.lastShortPollDuration)
		if sub.startTime != reply.StartTime {
			sub.generationCount = 0 // Sub has restarted: force a full poll.
		}
	} else {
		if err := fs.RebuildInodePointers(); err != nil {
			sub.status = statusFailedToPoll
			logger.Printf("Error building pointers for: %s %s\n",
				sub.hostname, err)
			return
		}
		fs.BuildInodeToFilenamesTable()
		fs.BuildEntryMap()
		sub.fileSystem = fs
		sub.objectCache = reply.ObjectCache
		sub.generationCount = reply.GenerationCount
		sub.lastFullPollDuration =
			sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
		fullPollDistribution.Add(sub.lastFullPollDuration)
		logger.Printf("Polled: %s, GenerationCount=%d\n",
			sub.hostname, reply.GenerationCount)
	}
	sub.startTime = reply.StartTime
	sub.pollTime = reply.PollTime
	if reply.FetchInProgress {
		sub.status = statusFetching
		return
	}
	if reply.UpdateInProgress {
		sub.status = statusUpdating
		return
	}
	if reply.GenerationCount < 1 {
		sub.status = statusSubNotReady
		return
	}
	if previousStatus == statusFetching && reply.LastFetchError != nil {
		sub.status = statusFailedToFetch
		if sub.fileSystem == nil {
			sub.generationCount = 0 // Force a full poll next cycle.
			return
		}
	}
	if previousStatus == statusUpdating && reply.LastUpdateError != nil {
		sub.status = statusFailedToUpdate
		if sub.fileSystem == nil {
			sub.generationCount = 0 // Force a full poll next cycle.
			return
		}
	}
	if !haveImage {
		sub.status = statusImageNotReady
		return
	}
	if sub.generationCountAtChangeStart == sub.generationCount {
		sub.status = statusWaitingForNextFullPoll
		return
	}
	if sub.fileSystem == nil {
		sub.status = previousStatus
		if previousStatus != statusSynced {
			// Force a full poll next cycle so that we can see the state of the
			// sub and recompute/retry what needs to be done.
			sub.generationCount = 0
		}
		return
	}
	if idle, status := sub.fetchMissingObjects(srpcClient,
		sub.requiredImage); !idle {
		sub.status = status
		sub.reclaim()
		return
	}
	sub.status = statusComputingUpdate
	if idle, status := sub.sendUpdate(srpcClient); !idle {
		sub.status = status
		sub.reclaim()
		return
	}
	if idle, status := sub.fetchMissingObjects(srpcClient,
		sub.plannedImage); !idle {
		if status != statusImageNotReady {
			sub.status = status
			sub.reclaim()
			return
		}
	}
	sub.status = statusSynced
	sub.cleanup(srpcClient, sub.plannedImage)
	sub.reclaim()
}

func (sub *Sub) reclaim() {
	sub.fileSystem = nil  // Mark memory for reclaim.
	sub.objectCache = nil // Mark memory for reclaim.
	runtime.GC()          // Reclaim now.
}

// Returns true if all required objects are available.
func (sub *Sub) fetchMissingObjects(srpcClient *srpc.Client, imageName string) (
	bool, subStatus) {
	image := sub.herd.getImageNoError(imageName)
	if image == nil {
		return false, statusImageNotReady
	}
	missingObjects := make(map[hash.Hash]struct{})
	for _, inode := range image.FileSystem.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				missingObjects[inode.Hash] = struct{}{}
			}
		}
	}
	for _, hash := range sub.objectCache {
		delete(missingObjects, hash)
	}
	for _, inode := range sub.fileSystem.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				delete(missingObjects, inode.Hash)
			}
		}
	}
	if len(missingObjects) < 1 {
		return true, statusSynced
	}
	logger := sub.herd.logger
	logger.Printf("Calling %s.Fetch() for: %d objects\n",
		sub.hostname, len(missingObjects))
	var request subproto.FetchRequest
	var reply subproto.FetchResponse
	request.ServerAddress = sub.herd.imageServerAddress
	for hash := range missingObjects {
		request.Hashes = append(request.Hashes, hash)
	}
	if err := client.CallFetch(srpcClient, request, &reply); err != nil {
		logger.Printf("Error calling %s.Fetch()\t%s\n", sub.hostname, err)
		if err == srpc.ErrorAccessToMethodDenied {
			return false, statusFetchDenied
		}
		return false, statusFailedToFetch
	}
	sub.generationCountAtChangeStart = sub.generationCount
	return false, statusFetching
}

// Returns true if no update needs to be performed.
func (sub *Sub) sendUpdate(srpcClient *srpc.Client) (bool, subStatus) {
	logger := sub.herd.logger
	var request subproto.UpdateRequest
	var reply subproto.UpdateResponse
	if sub.buildUpdateRequest(&request) {
		return true, statusSynced
	}
	sub.status = statusSendingUpdate
	if err := client.CallUpdate(srpcClient, request, &reply); err != nil {
		logger.Printf("Error calling %s:Subd.Update()\t%s\n", sub.hostname, err)
		if err == srpc.ErrorAccessToMethodDenied {
			return false, statusUpdateDenied
		}
		return false, statusFailedToUpdate
	}
	sub.generationCountAtChangeStart = sub.generationCount
	return false, statusUpdating
}

func (sub *Sub) cleanup(srpcClient *srpc.Client, plannedImageName string) {
	logger := sub.herd.logger
	unusedObjects := make(map[hash.Hash]bool)
	for _, hash := range sub.objectCache {
		unusedObjects[hash] = false // Potential cleanup candidate.
	}
	for _, inode := range sub.fileSystem.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				if _, ok := unusedObjects[inode.Hash]; ok {
					unusedObjects[inode.Hash] = true // Must clean this one up.
				}
			}
		}
	}
	image := sub.herd.getImageNoError(plannedImageName)
	if image != nil {
		for _, inode := range image.FileSystem.InodeTable {
			if inode, ok := inode.(*filesystem.RegularInode); ok {
				if inode.Size > 0 {
					if clean, ok := unusedObjects[inode.Hash]; !clean && ok {
						delete(unusedObjects, inode.Hash)
					}
				}
			}
		}
	}
	if len(unusedObjects) < 1 {
		return
	}
	var request subproto.CleanupRequest
	var reply subproto.CleanupResponse
	request.Hashes = make([]hash.Hash, 0, len(unusedObjects))
	for hash := range unusedObjects {
		request.Hashes = append(request.Hashes, hash)
	}
	if err := client.CallCleanup(srpcClient, request, &reply); err != nil {
		logger.Printf("Error calling %s:Subd.Cleanup()\t%s\n",
			sub.hostname, err)
	} else {
		sub.generationCountAtChangeStart = sub.generationCount
	}
}
<commit_msg>Only wait for next full poll if no poll data available.<commit_after>package herd

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/srpc"
	subproto "github.com/Symantec/Dominator/proto/sub"
	"github.com/Symantec/Dominator/sub/client"
	"net"
	"runtime"
	"strings"
	"time"
)

var (
	subConnectTimeout = flag.Uint("subConnectTimeout", 15,
		"Timeout in seconds for sub connections. If zero, OS timeout is used")
	logUnknownSubConnectErrors = flag.Bool("logUnknownSubConnectErrors", false,
		"If true, log unknown sub connection errors")
)

func (sub *Sub) tryMakeBusy() bool {
	sub.busyMutex.Lock()
	defer sub.busyMutex.Unlock()
	if sub.busy {
		return false
	}
	sub.busyStartTime = time.Now()
	sub.busy = true
	return true
}

func (sub *Sub) makeUnbusy() {
	sub.busyMutex.Lock()
	defer sub.busyMutex.Unlock()
	sub.busyStopTime = time.Now()
	sub.busy = false
}

func (sub *Sub) connectAndPoll() {
	previousStatus := sub.status
	sub.status = statusConnecting
	hostname := strings.SplitN(sub.hostname, "*", 2)[0]
	address := fmt.Sprintf("%s:%d", hostname, constants.SubPortNumber)
	sub.lastConnectionStartTime = time.Now()
	srpcClient, err := srpc.DialHTTP("tcp", address,
		time.Second*time.Duration(*subConnectTimeout))
	dialReturnedTime := time.Now()
	if err != nil {
		sub.pollTime = time.Time{}
		if err, ok := err.(*net.OpError); ok {
			if _, ok := err.Err.(*net.DNSError); ok {
				sub.status = statusDNSError
				return
			}
			if err.Timeout() {
				sub.status = statusConnectTimeout
				return
			}
		}
		if err == srpc.ErrorConnectionRefused {
			sub.status = statusConnectionRefused
			return
		}
		if err == srpc.ErrorNoRouteToHost {
			sub.status = statusNoRouteToHost
			return
		}
		if err == srpc.ErrorMissingCertificate {
			sub.lastReachableTime = dialReturnedTime
			sub.status = statusMissingCertificate
			return
		}
		if err == srpc.ErrorBadCertificate {
			sub.lastReachableTime = dialReturnedTime
			sub.status = statusBadCertificate
			return
		}
		sub.status = statusFailedToConnect
		if *logUnknownSubConnectErrors {
			sub.herd.logger.Println(err)
		}
		return
	}
	defer srpcClient.Close()
	sub.status = statusWaitingToPoll
	sub.lastReachableTime = dialReturnedTime
	sub.lastConnectionSucceededTime = dialReturnedTime
	sub.lastConnectDuration =
		sub.lastConnectionSucceededTime.Sub(sub.lastConnectionStartTime)
	connectDistribution.Add(sub.lastConnectDuration)
	sub.herd.pollSemaphore <- true
	sub.status = statusPolling
	sub.poll(srpcClient, previousStatus)
	<-sub.herd.pollSemaphore
}

func (sub *Sub) poll(srpcClient *srpc.Client, previousStatus subStatus) {
	// If the planned image has just become available, force a full poll.
	if previousStatus == statusSynced &&
		!sub.havePlannedImage &&
		sub.herd.getImageNoError(sub.plannedImage) != nil {
		sub.havePlannedImage = true
		sub.generationCount = 0 // Force a full poll.
	}
	var request subproto.PollRequest
	request.HaveGeneration = sub.generationCount
	var reply subproto.PollResponse
	sub.lastPollStartTime = time.Now()
	haveImage := false
	if sub.herd.getImageNoError(sub.requiredImage) == nil {
		request.ShortPollOnly = true
	} else {
		haveImage = true
	}
	logger := sub.herd.logger
	if err := client.CallPoll(srpcClient, request, &reply); err != nil {
		sub.pollTime = time.Time{}
		if err == srpc.ErrorAccessToMethodDenied {
			sub.status = statusPollDenied
		} else {
			sub.status = statusFailedToPoll
		}
		logger.Printf("Error calling %s.Poll()\t%s\n", sub.hostname, err)
		return
	}
	sub.lastPollSucceededTime = time.Now()
	if reply.GenerationCount == 0 {
		sub.reclaim()
		sub.generationCount = 0
	}
	if fs := reply.FileSystem; fs == nil {
		sub.lastShortPollDuration =
			sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
		shortPollDistribution.Add(sub.lastShortPollDuration)
		if sub.startTime != reply.StartTime {
			sub.generationCount = 0 // Sub has restarted: force a full poll.
		}
	} else {
		if err := fs.RebuildInodePointers(); err != nil {
			sub.status = statusFailedToPoll
			logger.Printf("Error building pointers for: %s %s\n",
				sub.hostname, err)
			return
		}
		fs.BuildInodeToFilenamesTable()
		fs.BuildEntryMap()
		sub.fileSystem = fs
		sub.objectCache = reply.ObjectCache
		sub.generationCount = reply.GenerationCount
		sub.lastFullPollDuration =
			sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
		fullPollDistribution.Add(sub.lastFullPollDuration)
		logger.Printf("Polled: %s, GenerationCount=%d\n",
			sub.hostname, reply.GenerationCount)
	}
	sub.startTime = reply.StartTime
	sub.pollTime = reply.PollTime
	if reply.FetchInProgress {
		sub.status = statusFetching
		return
	}
	if reply.UpdateInProgress {
		sub.status = statusUpdating
		return
	}
	if reply.GenerationCount < 1 {
		sub.status = statusSubNotReady
		return
	}
	if previousStatus == statusFetching && reply.LastFetchError != nil {
		sub.status = statusFailedToFetch
		if sub.fileSystem == nil {
			sub.generationCount = 0 // Force a full poll next cycle.
			return
		}
	}
	if previousStatus == statusUpdating && reply.LastUpdateError != nil {
		sub.status = statusFailedToUpdate
		if sub.fileSystem == nil {
			sub.generationCount = 0 // Force a full poll next cycle.
			return
		}
	}
	if !haveImage {
		sub.status = statusImageNotReady
		return
	}
	if sub.fileSystem == nil {
		if sub.generationCountAtChangeStart == sub.generationCount {
			sub.status = statusWaitingForNextFullPoll
			return
		}
		sub.status = previousStatus
		if previousStatus != statusSynced {
			// Force a full poll next cycle so that we can see the state of the
			// sub and recompute/retry what needs to be done.
			sub.generationCount = 0
		}
		return
	}
	if idle, status := sub.fetchMissingObjects(srpcClient,
		sub.requiredImage); !idle {
		sub.status = status
		sub.reclaim()
		return
	}
	sub.status = statusComputingUpdate
	if idle, status := sub.sendUpdate(srpcClient); !idle {
		sub.status = status
		sub.reclaim()
		return
	}
	if idle, status := sub.fetchMissingObjects(srpcClient,
		sub.plannedImage); !idle {
		if status != statusImageNotReady {
			sub.status = status
			sub.reclaim()
			return
		}
	}
	sub.status = statusSynced
	sub.cleanup(srpcClient, sub.plannedImage)
	sub.reclaim()
}

func (sub *Sub) reclaim() {
	sub.fileSystem = nil  // Mark memory for reclaim.
	sub.objectCache = nil // Mark memory for reclaim.
	runtime.GC()          // Reclaim now.
}

// Returns true if all required objects are available.
func (sub *Sub) fetchMissingObjects(srpcClient *srpc.Client, imageName string) (
	bool, subStatus) {
	image := sub.herd.getImageNoError(imageName)
	if image == nil {
		return false, statusImageNotReady
	}
	missingObjects := make(map[hash.Hash]struct{})
	for _, inode := range image.FileSystem.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				missingObjects[inode.Hash] = struct{}{}
			}
		}
	}
	for _, hash := range sub.objectCache {
		delete(missingObjects, hash)
	}
	for _, inode := range sub.fileSystem.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				delete(missingObjects, inode.Hash)
			}
		}
	}
	if len(missingObjects) < 1 {
		return true, statusSynced
	}
	logger := sub.herd.logger
	logger.Printf("Calling %s.Fetch() for: %d objects\n",
		sub.hostname, len(missingObjects))
	var request subproto.FetchRequest
	var reply subproto.FetchResponse
	request.ServerAddress = sub.herd.imageServerAddress
	for hash := range missingObjects {
		request.Hashes = append(request.Hashes, hash)
	}
	if err := client.CallFetch(srpcClient, request, &reply); err != nil {
		logger.Printf("Error calling %s.Fetch()\t%s\n", sub.hostname, err)
		if err == srpc.ErrorAccessToMethodDenied {
			return false, statusFetchDenied
		}
		return false, statusFailedToFetch
	}
	sub.generationCountAtChangeStart = sub.generationCount
	return false, statusFetching
}

// Returns true if no update needs to be performed.
func (sub *Sub) sendUpdate(srpcClient *srpc.Client) (bool, subStatus) {
	logger := sub.herd.logger
	var request subproto.UpdateRequest
	var reply subproto.UpdateResponse
	if sub.buildUpdateRequest(&request) {
		return true, statusSynced
	}
	sub.status = statusSendingUpdate
	if err := client.CallUpdate(srpcClient, request, &reply); err != nil {
		logger.Printf("Error calling %s:Subd.Update()\t%s\n", sub.hostname, err)
		if err == srpc.ErrorAccessToMethodDenied {
			return false, statusUpdateDenied
		}
		return false, statusFailedToUpdate
	}
	sub.generationCountAtChangeStart = sub.generationCount
	return false, statusUpdating
}

func (sub *Sub) cleanup(srpcClient *srpc.Client, plannedImageName string) {
	logger := sub.herd.logger
	unusedObjects := make(map[hash.Hash]bool)
	for _, hash := range sub.objectCache {
		unusedObjects[hash] = false // Potential cleanup candidate.
	}
	for _, inode := range sub.fileSystem.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				if _, ok := unusedObjects[inode.Hash]; ok {
					unusedObjects[inode.Hash] = true // Must clean this one up.
				}
			}
		}
	}
	image := sub.herd.getImageNoError(plannedImageName)
	if image != nil {
		for _, inode := range image.FileSystem.InodeTable {
			if inode, ok := inode.(*filesystem.RegularInode); ok {
				if inode.Size > 0 {
					if clean, ok := unusedObjects[inode.Hash]; !clean && ok {
						delete(unusedObjects, inode.Hash)
					}
				}
			}
		}
	}
	if len(unusedObjects) < 1 {
		return
	}
	var request subproto.CleanupRequest
	var reply subproto.CleanupResponse
	request.Hashes = make([]hash.Hash, 0, len(unusedObjects))
	for hash := range unusedObjects {
		request.Hashes = append(request.Hashes, hash)
	}
	if err := client.CallCleanup(srpcClient, request, &reply); err != nil {
		logger.Printf("Error calling %s:Subd.Cleanup()\t%s\n",
			sub.hostname, err)
	} else {
		sub.generationCountAtChangeStart = sub.generationCount
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package walk

import (
	"unsafe"
	"syscall"
	"fmt"
)

import . "github.com/lxn/go-winapi"

var logViewOrigWndProcPtr uintptr
var _ subclassedWidget = &LogView{}

type LogView struct {
	WidgetBase
	logChan chan string
}

func NewLogView(parent Container) (*LogView, error) {
	lc := make(chan string, 1024)
	te := &LogView{logChan:lc}

	if err := initChildWidget(
		te,
		parent,
		"EDIT",
		WS_TABSTOP|WS_VISIBLE|WS_VSCROLL|ES_MULTILINE|ES_WANTRETURN,
		WS_EX_CLIENTEDGE); err != nil {
		return nil, err
	}
	te.setReadOnly(true)
	SendMessage(te.hWnd, EM_SETLIMITTEXT, 4294967295, 0)
	return te, nil
}

func (*LogView) origWndProcPtr() uintptr {
	return textEditOrigWndProcPtr
}

func (*LogView) setOrigWndProcPtr(ptr uintptr) {
	textEditOrigWndProcPtr = ptr
}

func (*LogView) LayoutFlags() LayoutFlags {
	return ShrinkableHorz | ShrinkableVert | GrowableHorz | GrowableVert | GreedyHorz | GreedyVert
}

func (te *LogView) MinSizeHint() Size {
	return te.dialogBaseUnitsToPixels(Size{20, 12})
}

func (te *LogView) SizeHint() Size {
	return Size{100, 100}
}

func (te *LogView) setTextSelection(start, end int) {
	SendMessage(te.hWnd, EM_SETSEL, uintptr(start), uintptr(end))
}

func (te *LogView) textLength() int{
	return int(SendMessage(te.hWnd, 0x000E, uintptr(0), uintptr(0)))
}

func (te *LogView) AppendText(value string) {
	//fmt.Println("AppendText=", value)
	textLength := te.textLength()
	//fmt.Println("TextLength=", textLength)
	te.setTextSelection(textLength, textLength)
	SendMessage(te.hWnd, EM_REPLACESEL, 0, uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(value))))
	//fmt.Println("Ret=", r)
}

func (te *LogView) setReadOnly(readOnly bool) error {
	if 0 == SendMessage(te.hWnd, EM_SETREADONLY, uintptr(BoolToBOOL(readOnly)), 0) {
		return newError("SendMessage(EM_SETREADONLY)")
	}

	return nil
}

func (te *LogView) PostAppendText(value string){
	fmt.Println("PostAppendText", value)
	te.logChan <- value
	PostMessage(te.hWnd, TEM_APPENDTEXT, 0, 0)
}

func (te *LogView)  Write(p []byte)(int, error){
	te.PostAppendText(string(p) + "\r\n")
	return len(p), nil
}

func (te *LogView) wndProc(hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr {
	switch msg {
	case WM_GETDLGCODE:
		if wParam == VK_RETURN {
			return DLGC_WANTALLKEYS
		}

		return DLGC_HASSETSEL | DLGC_WANTARROWS | DLGC_WANTCHARS
	case TEM_APPENDTEXT:
		select {
		case value := <- te.logChan:
			te.AppendText(value)
		default:
			return 0
		}
	}

	return te.WidgetBase.wndProc(hwnd, msg, wParam, lParam)
}
<commit_msg>remove unneeded comments<commit_after>// Copyright 2012 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package walk

import (
	"unsafe"
	"syscall"
)

import . "github.com/lxn/go-winapi"

var logViewOrigWndProcPtr uintptr
var _ subclassedWidget = &LogView{}

type LogView struct {
	WidgetBase
	logChan chan string
}

func NewLogView(parent Container) (*LogView, error) {
	lc := make(chan string, 1024)
	te := &LogView{logChan:lc}

	if err := initChildWidget(
		te,
		parent,
		"EDIT",
		WS_TABSTOP|WS_VISIBLE|WS_VSCROLL|ES_MULTILINE|ES_WANTRETURN,
		WS_EX_CLIENTEDGE); err != nil {
		return nil, err
	}
	te.setReadOnly(true)
	SendMessage(te.hWnd, EM_SETLIMITTEXT, 4294967295, 0)
	return te, nil
}

func (*LogView) origWndProcPtr() uintptr {
	return textEditOrigWndProcPtr
}

func (*LogView) setOrigWndProcPtr(ptr uintptr) {
	textEditOrigWndProcPtr = ptr
}

func (*LogView) LayoutFlags() LayoutFlags {
	return ShrinkableHorz | ShrinkableVert | GrowableHorz | GrowableVert | GreedyHorz | GreedyVert
}

func (te *LogView) MinSizeHint() Size {
	return te.dialogBaseUnitsToPixels(Size{20, 12})
}

func (te *LogView) SizeHint() Size {
	return Size{100, 100}
}

func (te *LogView) setTextSelection(start, end int) {
	SendMessage(te.hWnd, EM_SETSEL, uintptr(start), uintptr(end))
}

func (te *LogView) textLength() int{
	return int(SendMessage(te.hWnd, 0x000E, uintptr(0), uintptr(0)))
}

func (te *LogView) AppendText(value string) {
	textLength := te.textLength()
	te.setTextSelection(textLength, textLength)
	SendMessage(te.hWnd, EM_REPLACESEL, 0, uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(value))))
}

func (te *LogView) setReadOnly(readOnly bool) error {
	if 0 == SendMessage(te.hWnd, EM_SETREADONLY, uintptr(BoolToBOOL(readOnly)), 0) {
		return newError("SendMessage(EM_SETREADONLY)")
	}

	return nil
}

func (te *LogView) PostAppendText(value string){
	te.logChan <- value
	PostMessage(te.hWnd, TEM_APPENDTEXT, 0, 0)
}

func (te *LogView)  Write(p []byte)(int, error){
	te.PostAppendText(string(p) + "\r\n")
	return len(p), nil
}

func (te *LogView) wndProc(hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr {
	switch msg {
	case WM_GETDLGCODE:
		if wParam == VK_RETURN {
			return DLGC_WANTALLKEYS
		}

		return DLGC_HASSETSEL | DLGC_WANTARROWS | DLGC_WANTCHARS
	case TEM_APPENDTEXT:
		select {
		case value := <- te.logChan:
			te.AppendText(value)
		default:
			return 0
		}
	}

	return te.WidgetBase.wndProc(hwnd, msg, wParam, lParam)
}
<|endoftext|>


<commit_before>package los

import (
	"runtime"

	"github.com/phil-mansfield/gotetra/render/io"
	rGeom "github.com/phil-mansfield/gotetra/render/geom"
	"github.com/phil-mansfield/gotetra/los/geom"
)

type Buffers struct {
	xs []rGeom.Vec
	ts []geom.Tetra
	ss []geom.Sphere
	rhos []float64
	intr []bool
	bufHs []HaloProfiles
}

func NewBuffers(file string, hd *io.SheetHeader) *Buffers {
	buf := new(Buffers)

    sw := hd.SegmentWidth
    buf.xs = make([]rGeom.Vec, hd.GridCount)
    buf.ts = make([]geom.Tetra, 6*sw*sw*sw)
    buf.ss = make([]geom.Sphere, 6*sw*sw*sw)
    buf.rhos = make([]float64, 6*sw*sw*sw)
	buf.intr = make([]bool, 6*sw*sw*sw)

	buf.Read(file, hd)
	return buf
}

func (buf *Buffers) ParallelRead(file string, hd *io.SheetHeader) {
	workers := runtime.NumCPU()
	runtime.GOMAXPROCS(workers)
	buf.read(file, hd, workers)
}

func (buf *Buffers) Read(file string, hd *io.SheetHeader) {
	buf.read(file, hd, 1)
}

func (buf *Buffers) read(file string, hd *io.SheetHeader, workers int) {
	io.ReadSheetPositionsAt(file, buf.xs)
	tw := float32(hd.TotalWidth)
	// This can only be parallelized if we sychronize afterwards. This
	// is insignificant compared to the serial I/O time.
	for i := range buf.xs {
		for j := 0; j < 3; j++ {
			if buf.xs[i][j] < hd.Origin[j] {
				buf.xs[i][j] += tw
			}
		}
	}

	out := make(chan int, workers)
	for id := 0; id < workers - 1; id++ {
		go buf.chanRead(hd, id, workers, out)
	}
	buf.chanRead(hd, workers - 1, workers, out)

	for i := 0; i < workers; i++ { <- out }
}

func (buf *Buffers) chanRead(
	hd *io.SheetHeader, id, workers int, out chan<- int,
) {
	// Remember: Grid -> All particles; Segment -> Particles that can be turned
	// into tetrahedra.
	n := hd.SegmentWidth*hd.SegmentWidth*hd.SegmentWidth
	tw := hd.TotalWidth
	tFactor := tw*tw*tw / float64(hd.Count * 6)
	idxBuf := new(rGeom.TetraIdxs)

	jump := int64(workers)
	for segIdx := int64(id); segIdx < n; segIdx += jump {
		x, y, z := coords(segIdx, hd.SegmentWidth)
		for dir := int64(0); dir < 6; dir++ {
			ti := 6 * segIdx + dir
			idxBuf.InitCartesian(x, y, z, hd.GridWidth, int(dir))
			unpackTetra(idxBuf, buf.xs, &buf.ts[ti])
			buf.ts[ti].Orient(+1)

			buf.rhos[ti] = tFactor / buf.ts[ti].Volume()

			buf.ts[ti].BoundingSphere(&buf.ss[ti])
		}
	}

	out <- id
}

func (buf *Buffers) ParallelDensity(h *HaloProfiles) {
	workers := runtime.NumCPU()
	out := make(chan int, workers)

	for id := 0; id < workers - 1; id++ {
		go buf.chanIntersect(h, id, workers, out)
	}
	buf.chanIntersect(h, workers - 1, workers, out)
	for i := 0; i < workers; i++ { <-out }

	if workers > len(h.rs) { workers = len(h.rs) }
	for id := 0; id < workers - 1; id++ {
		go buf.chanDensity(h, id, workers, out)
	}
	buf.chanDensity(h, workers - 1, workers, out)
	for i := 0; i < workers; i++ { <-out }
}

func (buf *Buffers) chanDensity(
	h *HaloProfiles, id, workers int, out chan <- int,
) {
	for ri := id; ri < len(h.rs); ri += workers {
		r := &h.rs[ri]
		for ti := 0; ti < len(buf.ts); ti++ {
			if buf.intr[ti] { r.Density(&buf.ts[ti], buf.rhos[ti]) }
		}
	}
	out <- id
}

func (buf *Buffers) chanIntersect(
	h *HaloProfiles, id, workers int, out chan <- int,
) {
	bufLen := len(buf.ts) / workers
	bufStart, bufEnd := id * bufLen, (id + 1) * bufLen
	if id == workers - 1 { bufEnd = len(buf.ts) }
	for i := bufStart; i < bufEnd; i++ {
		buf.intr[i] = h.Sphere.SphereIntersect(&buf.ss[i]) &&
			!h.minSphere.TetraContain(&buf.ts[i])
	}
	out <- id
}

func coords(idx, cells int64) (x, y, z int64) {
    x = idx % cells
    y = (idx % (cells * cells)) / cells
    z = idx / (cells * cells)
    return x, y, z
}

func index(x, y, z, cells int64) int64 {
    return x + y * cells + z * cells * cells
}

func unpackTetra(idxs *rGeom.TetraIdxs, xs []rGeom.Vec, t *geom.Tetra) {
    for i := 0; i < 4; i++ {
		t[i] = geom.Vec(xs[idxs[i]])
    }
}

// WrapHalo updates the coordinates of a slice of HaloProfiles so that they
// as close to the given sheet as periodic boundary conditions will allow.
func WrapHalo(hps []*HaloProfiles, hd *io.SheetHeader) {
	tw := float32(hd.TotalWidth)
	newC := &geom.Vec{}
	for i := range hps {
		h := hps[i]
		for j := 0; j < 3; j++ {
			if h.cCopy[j] + h.R < hd.Origin[j] {
				newC[j] = h.cCopy[j] + tw
			} else {
				newC[j] = h.cCopy[j]
			}
		}
		h.ChangeCenter(newC)
	}
}
<commit_msg>Changed chanDensity to work on contiguous blocks instead of via jumps.<commit_after>package los

import (
	"runtime"

	"github.com/phil-mansfield/gotetra/render/io"
	rGeom "github.com/phil-mansfield/gotetra/render/geom"
	"github.com/phil-mansfield/gotetra/los/geom"
)

type Buffers struct {
	xs []rGeom.Vec
	ts []geom.Tetra
	ss []geom.Sphere
	rhos []float64
	intr []bool
	bufHs []HaloProfiles
}

func NewBuffers(file string, hd *io.SheetHeader) *Buffers {
	buf := new(Buffers)

    sw := hd.SegmentWidth
    buf.xs = make([]rGeom.Vec, hd.GridCount)
    buf.ts = make([]geom.Tetra, 6*sw*sw*sw)
    buf.ss = make([]geom.Sphere, 6*sw*sw*sw)
    buf.rhos = make([]float64, 6*sw*sw*sw)
	buf.intr = make([]bool, 6*sw*sw*sw)

	buf.Read(file, hd)
	return buf
}

func (buf *Buffers) ParallelRead(file string, hd *io.SheetHeader) {
	workers := runtime.NumCPU()
	runtime.GOMAXPROCS(workers)
	buf.read(file, hd, workers)
}

func (buf *Buffers) Read(file string, hd *io.SheetHeader) {
	buf.read(file, hd, 1)
}

func (buf *Buffers) read(file string, hd *io.SheetHeader, workers int) {
	io.ReadSheetPositionsAt(file, buf.xs)
	tw := float32(hd.TotalWidth)
	// This can only be parallelized if we sychronize afterwards. This
	// is insignificant compared to the serial I/O time.
	for i := range buf.xs {
		for j := 0; j < 3; j++ {
			if buf.xs[i][j] < hd.Origin[j] {
				buf.xs[i][j] += tw
			}
		}
	}

	out := make(chan int, workers)
	for id := 0; id < workers - 1; id++ {
		go buf.chanRead(hd, id, workers, out)
	}
	buf.chanRead(hd, workers - 1, workers, out)

	for i := 0; i < workers; i++ { <- out }
}

func (buf *Buffers) chanRead(
	hd *io.SheetHeader, id, workers int, out chan<- int,
) {
	// Remember: Grid -> All particles; Segment -> Particles that can be turned
	// into tetrahedra.
	n := hd.SegmentWidth*hd.SegmentWidth*hd.SegmentWidth
	tw := hd.TotalWidth
	tFactor := tw*tw*tw / float64(hd.Count * 6)
	idxBuf := new(rGeom.TetraIdxs)

	jump := int64(workers)
	for segIdx := int64(id); segIdx < n; segIdx += jump {
		x, y, z := coords(segIdx, hd.SegmentWidth)
		for dir := int64(0); dir < 6; dir++ {
			ti := 6 * segIdx + dir
			idxBuf.InitCartesian(x, y, z, hd.GridWidth, int(dir))
			unpackTetra(idxBuf, buf.xs, &buf.ts[ti])
			buf.ts[ti].Orient(+1)

			buf.rhos[ti] = tFactor / buf.ts[ti].Volume()

			buf.ts[ti].BoundingSphere(&buf.ss[ti])
		}
	}

	out <- id
}

func (buf *Buffers) ParallelDensity(h *HaloProfiles) {
	workers := runtime.NumCPU()
	out := make(chan int, workers)

	for id := 0; id < workers - 1; id++ {
		go buf.chanIntersect(h, id, workers, out)
	}
	buf.chanIntersect(h, workers - 1, workers, out)
	for i := 0; i < workers; i++ { <-out }

	if workers > len(h.rs) { workers = len(h.rs) }
	for id := 0; id < workers - 1; id++ {
		go buf.chanDensity(h, id, workers, out)
	}
	buf.chanDensity(h, workers - 1, workers, out)
	for i := 0; i < workers; i++ { <-out }
}

func (buf *Buffers) chanDensity(
	h *HaloProfiles, id, workers int, out chan <- int,
) {
	bufLen := len(buf.ts) / workers
	bufStart, bufEnd := id * bufLen, (id + 1) * bufLen
	if id == workers - 1 { bufEnd = len(buf.ts) }
	for ri := bufStart; ri < bufEnd; ri++ {
		r := &h.rs[ri]
		for ti := 0; ti < len(buf.ts); ti++ {
			if buf.intr[ti] { r.Density(&buf.ts[ti], buf.rhos[ti]) }
		}
	}
	out <- id
}

func (buf *Buffers) chanIntersect(
	h *HaloProfiles, id, workers int, out chan <- int,
) {
	bufLen := len(buf.ts) / workers
	bufStart, bufEnd := id * bufLen, (id + 1) * bufLen
	if id == workers - 1 { bufEnd = len(buf.ts) }
	for i := bufStart; i < bufEnd; i++ {
		buf.intr[i] = h.Sphere.SphereIntersect(&buf.ss[i]) &&
			!h.minSphere.TetraContain(&buf.ts[i])
	}
	out <- id
}

func coords(idx, cells int64) (x, y, z int64) {
    x = idx % cells
    y = (idx % (cells * cells)) / cells
    z = idx / (cells * cells)
    return x, y, z
}

func index(x, y, z, cells int64) int64 {
    return x + y * cells + z * cells * cells
}

func unpackTetra(idxs *rGeom.TetraIdxs, xs []rGeom.Vec, t *geom.Tetra) {
    for i := 0; i < 4; i++ {
		t[i] = geom.Vec(xs[idxs[i]])
    }
}

// WrapHalo updates the coordinates of a slice of HaloProfiles so that they
// as close to the given sheet as periodic boundary conditions will allow.
func WrapHalo(hps []*HaloProfiles, hd *io.SheetHeader) {
	tw := float32(hd.TotalWidth)
	newC := &geom.Vec{}
	for i := range hps {
		h := hps[i]
		for j := 0; j < 3; j++ {
			if h.cCopy[j] + h.R < hd.Origin[j] {
				newC[j] = h.cCopy[j] + tw
			} else {
				newC[j] = h.cCopy[j]
			}
		}
		h.ChangeCenter(newC)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"strconv"

	. "github.com/pivotal-cf-experimental/switchboard"
)

var (
	pidfile = flag.String("pidfile", "", "The location for the pidfile")
	port    = flag.Uint("port", 3306, "Port to listen on")

	backendIp       = flag.String("backendIp", "", "IP address of backend")
	backendPort     = flag.Uint("backendPort", 3306, "Port of backend")
	healthcheckPort = flag.Uint("healthcheckPort", 9200, "Port for healthcheck endpoints")
)

func acceptClientConnection(l net.Listener) net.Conn {
	clientConn, err := l.Accept()
	if err != nil {
		log.Fatal("Error accepting client connection: %v", err)
	}
	return clientConn
}

func main() {
	flag.Parse()

	l, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", *port))
	defer l.Close()
	if err != nil {
		log.Fatal(fmt.Sprintf("Error listening on port %d: %v\n", *port, err.Error()))
	}

	err = ioutil.WriteFile(*pidfile, []byte(strconv.Itoa(os.Getpid())), 0644)
	if err != nil {
		log.Fatal(fmt.Sprintf("Cannot write pid to file: %s", *pidfile))
	}

	fmt.Printf("Proxy started on port %d\n", *port)
	fmt.Printf("Backend ipAddress: %s\n", *backendIp)
	fmt.Printf("Backend port: %d\n", *port)
	fmt.Printf("Healthcheck port: %d\n", *healthcheckPort)

	backend := NewBackend("backend1", *backendIp, *backendPort)

	healthcheck := NewHttpHealthCheck(*backendIp, *healthcheckPort)
	healthcheck.Start(backend.RemoveAndCloseAllBridges)

	for {
		clientConn := acceptClientConnection(l)
		defer clientConn.Close()

		backendConn, err := backend.Dial()
		if err != nil {
			log.Fatal("Error connection to backend: %s", err.Error())
		}
		defer backendConn.Close()

		bridge := NewConnectionBridge(clientConn, backendConn)
		backend.AddBridge(bridge)

		go func() {
			bridge.Connect()
			backend.RemoveBridge(bridge)
		}()
	}
}
<commit_msg>handle nil listener on error [#78749204]<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"strconv"

	. "github.com/pivotal-cf-experimental/switchboard"
)

var (
	pidfile = flag.String("pidfile", "", "The location for the pidfile")
	port    = flag.Uint("port", 3306, "Port to listen on")

	backendIp       = flag.String("backendIp", "", "IP address of backend")
	backendPort     = flag.Uint("backendPort", 3306, "Port of backend")
	healthcheckPort = flag.Uint("healthcheckPort", 9200, "Port for healthcheck endpoints")
)

func acceptClientConnection(l net.Listener) net.Conn {
	clientConn, err := l.Accept()
	if err != nil {
		log.Fatal("Error accepting client connection: %v", err)
	}
	return clientConn
}

func main() {
	flag.Parse()

	l, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", *port))
	if err != nil {
		log.Fatal(fmt.Sprintf("Error listening on port %d: %v\n", *port, err.Error()))
	}
	defer l.Close()

	err = ioutil.WriteFile(*pidfile, []byte(strconv.Itoa(os.Getpid())), 0644)
	if err != nil {
		log.Fatal(fmt.Sprintf("Cannot write pid to file: %s", *pidfile))
	}

	fmt.Printf("Proxy started on port %d\n", *port)
	fmt.Printf("Backend ipAddress: %s\n", *backendIp)
	fmt.Printf("Backend port: %d\n", *port)
	fmt.Printf("Healthcheck port: %d\n", *healthcheckPort)

	backend := NewBackend("backend1", *backendIp, *backendPort)

	healthcheck := NewHttpHealthCheck(*backendIp, *healthcheckPort)
	healthcheck.Start(backend.RemoveAndCloseAllBridges)

	for {
		clientConn := acceptClientConnection(l)
		defer clientConn.Close()

		backendConn, err := backend.Dial()
		if err != nil {
			log.Fatal("Error connection to backend: %s", err.Error())
		}
		defer backendConn.Close()

		bridge := NewConnectionBridge(clientConn, backendConn)
		backend.AddBridge(bridge)

		go func() {
			bridge.Connect()
			backend.RemoveBridge(bridge)
		}()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"strings"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/Sirupsen/logrus"
	"github.com/stripe/veneur"
)

var (
	configFile = flag.String("f", "", "The Veneur config file to read for settings.")
	hostport   = flag.String("hostport", "", "Hostname and port of destination. Must be used if config file is not present.")
	name       = flag.String("name", "", "Name of metric to report. Ex: daemontools.service.starts")
	gauge      = flag.Float64("gauge", 0, "Report a 'gauge' metric. Value must be float64.")
	timing     = flag.Duration("timing", time.Now().Sub(time.Now()), "Report a 'timing' metric. Value must be parseable by time.ParseDuration.")
	count      = flag.Int64("count", 0, "Report a 'count' metric. Value must be an integer.")
	tag        = flag.String("tag", "", "Tag(s) for metric, comma separated. Ex: service:airflow")
	debug      = flag.Bool("debug", false, "Turns on debug messages.")
)

// MinimalClient represents the functions that we call on Clients in veneur-emit.
type MinimalClient interface {
	Gauge(name string, value float64, tags []string, rate float64) error
	Count(name string, value int64, tags []string, rate float64) error
	Timing(name string, value time.Duration, tags []string, rate float64) error
}

func main() {
	passedFlags := flags()

	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
	}

	var config *veneur.Config
	if passedFlags["f"] {
		conf, err := veneur.ReadConfig(*configFile)
		if err != nil {
			logrus.WithError(err).Fatal("Error reading configuration file.")
		}
		config = &conf
	}

	addr, err := addr(passedFlags, config, hostport)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting destination address.")
	}
	logrus.Debugf("destination: %s", addr)

	var conn MinimalClient
	conn, err = statsd.New(addr)
	if err != nil {
		logrus.WithError(err).Fatal("Error!")
	}

	tags := tags(*tag)
	err = sendMetrics(conn, passedFlags, *name, tags)
	if err != nil {
		logrus.WithError(err).Fatal("Error sending metric(s).")
	}
}

func flags() map[string]bool {
	flag.Parse()
	// hacky way to detect which flags were *actually* set
	passedFlags := make(map[string]bool)
	flag.Visit(func(f *flag.Flag) {
		passedFlags[f.Name] = true
	})
	return passedFlags
}

func tags(tag string) []string {
	var tags []string
	for _, elem := range strings.Split(tag, ",") {
		tags = append(tags, elem)
	}
	return tags
}

func addr(passedFlags map[string]bool, conf *veneur.Config, hostport *string) (string, error) {
	addr := ""
	var err error
	if conf != nil {
		addr = conf.UdpAddress // TODO: conf.TcpAddress
	} else if passedFlags["hostport"] && hostport != nil {
		addr = *hostport
	} else {
		err = errors.New("you must either specify a Veneur config file or a valid hostport")
	}
	return addr, err
}

func sendMetrics(client MinimalClient, passedFlags map[string]bool, name string, tags []string) error {
	var err error
	if passedFlags["gauge"] {
		logrus.Debugf("Sending gauge '%s' -> %f", name, *gauge)
		err = client.Gauge(name, *gauge, tags, 1)
		if err != nil {
			return err
		}
	}
	if passedFlags["timing"] {
		logrus.Debugf("Sending timing '%s' -> %f", name, *timing)
		err = client.Timing(name, *timing, tags, 1)
		if err != nil {
			return err
		}
	}
	if passedFlags["count"] {
		logrus.Debugf("Sending count '%s' -> %d", name, *count)
		err = client.Count(name, *count, tags, 1)
		if err != nil {
			return err
		}
	}
	if !passedFlags["gauge"] && !passedFlags["timing"] && !passedFlags["timeinms"] && !passedFlags["count"] {
		logrus.Info("No metrics reported.")
	}
	return err
	// TODO: error on no metrics?
}
<commit_msg>Added doc link for timing<commit_after>package main

import (
	"errors"
	"flag"
	"strings"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/Sirupsen/logrus"
	"github.com/stripe/veneur"
)

var (
	configFile = flag.String("f", "", "The Veneur config file to read for settings.")
	hostport   = flag.String("hostport", "", "Hostname and port of destination. Must be used if config file is not present.")
	name       = flag.String("name", "", "Name of metric to report. Ex: daemontools.service.starts")
	gauge      = flag.Float64("gauge", 0, "Report a 'gauge' metric. Value must be float64.")
	timing     = flag.Duration("timing", time.Now().Sub(time.Now()), "Report a 'timing' metric. Value must be parseable by time.ParseDuration (https://golang.org/pkg/time/#ParseDuration).")
	count      = flag.Int64("count", 0, "Report a 'count' metric. Value must be an integer.")
	tag        = flag.String("tag", "", "Tag(s) for metric, comma separated. Ex: service:airflow")
	debug      = flag.Bool("debug", false, "Turns on debug messages.")
)

// MinimalClient represents the functions that we call on Clients in veneur-emit.
type MinimalClient interface {
	Gauge(name string, value float64, tags []string, rate float64) error
	Count(name string, value int64, tags []string, rate float64) error
	Timing(name string, value time.Duration, tags []string, rate float64) error
}

func main() {
	passedFlags := flags()

	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
	}

	var config *veneur.Config
	if passedFlags["f"] {
		conf, err := veneur.ReadConfig(*configFile)
		if err != nil {
			logrus.WithError(err).Fatal("Error reading configuration file.")
		}
		config = &conf
	}

	addr, err := addr(passedFlags, config, hostport)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting destination address.")
	}
	logrus.Debugf("destination: %s", addr)

	var conn MinimalClient
	conn, err = statsd.New(addr)
	if err != nil {
		logrus.WithError(err).Fatal("Error!")
	}

	tags := tags(*tag)
	err = sendMetrics(conn, passedFlags, *name, tags)
	if err != nil {
		logrus.WithError(err).Fatal("Error sending metric(s).")
	}
}

func flags() map[string]bool {
	flag.Parse()
	// hacky way to detect which flags were *actually* set
	passedFlags := make(map[string]bool)
	flag.Visit(func(f *flag.Flag) {
		passedFlags[f.Name] = true
	})
	return passedFlags
}

func tags(tag string) []string {
	var tags []string
	for _, elem := range strings.Split(tag, ",") {
		tags = append(tags, elem)
	}
	return tags
}

func addr(passedFlags map[string]bool, conf *veneur.Config, hostport *string) (string, error) {
	addr := ""
	var err error
	if conf != nil {
		addr = conf.UdpAddress // TODO: conf.TcpAddress
	} else if passedFlags["hostport"] && hostport != nil {
		addr = *hostport
	} else {
		err = errors.New("you must either specify a Veneur config file or a valid hostport")
	}
	return addr, err
}

func sendMetrics(client MinimalClient, passedFlags map[string]bool, name string, tags []string) error {
	var err error
	if passedFlags["gauge"] {
		logrus.Debugf("Sending gauge '%s' -> %f", name, *gauge)
		err = client.Gauge(name, *gauge, tags, 1)
		if err != nil {
			return err
		}
	}
	if passedFlags["timing"] {
		logrus.Debugf("Sending timing '%s' -> %f", name, *timing)
		err = client.Timing(name, *timing, tags, 1)
		if err != nil {
			return err
		}
	}
	if passedFlags["count"] {
		logrus.Debugf("Sending count '%s' -> %d", name, *count)
		err = client.Count(name, *count, tags, 1)
		if err != nil {
			return err
		}
	}
	if !passedFlags["gauge"] && !passedFlags["timing"] && !passedFlags["timeinms"] && !passedFlags["count"] {
		logrus.Info("No metrics reported.")
	}
	return err
	// TODO: error on no metrics?
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"net"
	"strings"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/Sirupsen/logrus"
	"github.com/gogo/protobuf/proto"
	"github.com/stripe/veneur"
	"github.com/stripe/veneur/ssf"
)

var (
	configFile = flag.String("f", "", "The Veneur config file to read for settings.")
	hostport   = flag.String("hostport", "", "Hostname and port of destination. Must be used if config file is not present.")
	name       = flag.String("name", "", "Name of metric to report. Ex: daemontools.service.starts")
	gauge      = flag.Float64("gauge", 0, "Report a 'gauge' metric. Value must be float64.")
	timing     = flag.Duration("timing", time.Now().Sub(time.Now()), "Report a 'timing' metric. Value must be parseable by time.ParseDuration (https://golang.org/pkg/time/#ParseDuration).")
	count      = flag.Int64("count", 0, "Report a 'count' metric. Value must be an integer.")
	tag        = flag.String("tag", "", "Tag(s) for metric, comma separated. Ex: service:airflow")
	debug      = flag.Bool("debug", false, "Turns on debug messages.")
)

// MinimalClient represents the functions that we call on Clients in veneur-emit.
type MinimalClient interface {
	Gauge(name string, value float64, tags []string, rate float64) error
	Count(name string, value int64, tags []string, rate float64) error
	Timing(name string, value time.Duration, tags []string, rate float64) error
}

// MinimalConn represents the functions that we call on connections in veneur-emit.
type MinimalConn interface {
	Write([]byte) (int, error)
}

func main() {
	passedFlags := flags()

	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
	}

	var config *veneur.Config
	if passedFlags["f"] {
		conf, err := veneur.ReadConfig(*configFile)
		if err != nil {
			logrus.WithError(err).Fatal("Error reading configuration file.")
		}
		config = &conf
	}

	addr, err := addr(passedFlags, config, hostport)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting destination address.")
	}
	logrus.Debugf("destination: %s", addr)

	var conn MinimalClient
	conn, err = statsd.New(addr)
	if err != nil {
		logrus.WithError(err).Fatal("Error!")
	}
	_ = conn

	var nconn net.Conn
	nconn, err = net.Dial("udp", addr)
	if err != nil {
		logrus.WithError(err).Fatal("Error!")
	}

	// tags := tags(*tag)

	var span *ssf.SSFSpan
	span, err = createMetrics(passedFlags, *name, *tag)
	if err != nil {
		logrus.WithError(err).Fatal("Error creating metric(s).")
	}

	err = sendSpan(nconn, span)
	if err != nil {
		logrus.WithError(err).Fatal("Error sending metric(s).")
	}
}

func flags() map[string]bool {
	flag.Parse()
	// hacky way to detect which flags were *actually* set
	passedFlags := make(map[string]bool)
	flag.Visit(func(f *flag.Flag) {
		passedFlags[f.Name] = true
	})
	return passedFlags
}

func tags(tag string) []string {
	var tags []string
	for _, elem := range strings.Split(tag, ",") {
		tags = append(tags, elem)
	}
	return tags
}

func addr(passedFlags map[string]bool, conf *veneur.Config, hostport *string) (string, error) {
	addr := ""
	var err error
	if conf != nil {
		addr = conf.UdpAddress // TODO: conf.TcpAddress
	} else if passedFlags["hostport"] && hostport != nil {
		addr = *hostport
	} else {
		err = errors.New("you must either specify a Veneur config file or a valid hostport")
	}
	return addr, err
}

func bareMetric(name string, tags string) *ssf.SSFSample {
	metric := &ssf.SSFSample{}
	metric.Name = name
	metric.Tags = make(map[string]string)
	for _, elem := range strings.Split(tags, ",") {
		tag := strings.Split(elem, ":")
		metric.Tags[tag[0]] = tag[1]
	}
	return metric
}

func createMetrics(passedFlags map[string]bool, name string, tags string) (*ssf.SSFSpan, error) {
	var err error
	span := &ssf.SSFSpan{}
	if passedFlags["gauge"] {
		logrus.Debugf("Sending gauge '%s' -> %f", name, *gauge)
		metric := bareMetric(name, tags)
		metric.Metric = ssf.SSFSample_GAUGE
		metric.Value = float32(*gauge)
		span.Metrics = append(span.Metrics, metric)
	}
	// TODO: figure out timing metrics
	if passedFlags["count"] {
		logrus.Debugf("Sending count '%s' -> %d", name, *count)
		metric := bareMetric(name, tags)
		metric.Metric = ssf.SSFSample_COUNTER
		metric.Value = float32(*count)
		span.Metrics = append(span.Metrics, metric)
	}
	return span, err
}

func sendSpan(conn MinimalConn, span *ssf.SSFSpan) error {
	var err error
	var data []byte
	data, err = proto.Marshal(span)
	if err != nil {
		return err
	}

	_, err = conn.Write(data)
	if err != nil {
		return err
	}
	return nil
}

func sendMetrics(client MinimalClient, passedFlags map[string]bool, name string, tags []string) error {
	var err error
	if passedFlags["gauge"] {
		logrus.Debugf("Sending gauge '%s' -> %f", name, *gauge)
		err = client.Gauge(name, *gauge, tags, 1)
		if err != nil {
			return err
		}
	}
	if passedFlags["timing"] {
		logrus.Debugf("Sending timing '%s' -> %f", name, *timing)
		err = client.Timing(name, *timing, tags, 1)
		if err != nil {
			return err
		}
	}
	if passedFlags["count"] {
		logrus.Debugf("Sending count '%s' -> %d", name, *count)
		err = client.Count(name, *count, tags, 1)
		if err != nil {
			return err
		}
	}
	if !passedFlags["gauge"] && !passedFlags["timing"] && !passedFlags["timeinms"] && !passedFlags["count"] {
		logrus.Info("No metrics reported.")
	}
	return err
	// TODO: error on no metrics?
}
<commit_msg>Bool for sending packets via SSF<commit_after>package main

import (
	"errors"
	"flag"
	"net"
	"strings"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/Sirupsen/logrus"
	"github.com/gogo/protobuf/proto"
	"github.com/stripe/veneur"
	"github.com/stripe/veneur/ssf"
)

var (
	configFile = flag.String("f", "", "The Veneur config file to read for settings.")
	hostport   = flag.String("hostport", "", "Hostname and port of destination. Must be used if config file is not present.")
	name       = flag.String("name", "", "Name of metric to report. Ex: daemontools.service.starts")
	gauge      = flag.Float64("gauge", 0, "Report a 'gauge' metric. Value must be float64.")
	timing     = flag.Duration("timing", time.Now().Sub(time.Now()), "Report a 'timing' metric. Value must be parseable by time.ParseDuration (https://golang.org/pkg/time/#ParseDuration).")
	count      = flag.Int64("count", 0, "Report a 'count' metric. Value must be an integer.")
	tag        = flag.String("tag", "", "Tag(s) for metric, comma separated. Ex: service:airflow")
	debug      = flag.Bool("debug", false, "Turns on debug messages.")
	toSSF      = flag.Bool("ssf", false, "Sends packets via SSF instead of StatsD. (https://github.com/stripe/veneur/blob/master/ssf/sample.proto)")
)

// MinimalClient represents the functions that we call on Clients in veneur-emit.
type MinimalClient interface {
	Gauge(name string, value float64, tags []string, rate float64) error
	Count(name string, value int64, tags []string, rate float64) error
	Timing(name string, value time.Duration, tags []string, rate float64) error
}

// MinimalConn represents the functions that we call on connections in veneur-emit.
type MinimalConn interface {
	Write([]byte) (int, error)
}

func main() {
	passedFlags := flags()

	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
	}

	var config *veneur.Config
	if passedFlags["f"] {
		conf, err := veneur.ReadConfig(*configFile)
		if err != nil {
			logrus.WithError(err).Fatal("Error reading configuration file.")
		}
		config = &conf
	}

	addr, err := addr(passedFlags, config, hostport)
	if err != nil {
		logrus.WithError(err).Fatal("Error getting destination address.")
	}
	logrus.Debugf("destination: %s", addr)

	if *toSSF {
		var nconn net.Conn
		nconn, err = net.Dial("udp", addr)
		if err != nil {
			logrus.WithError(err).Fatal("Error!")
		}

		var span *ssf.SSFSpan
		span, err = createMetrics(passedFlags, *name, *tag)
		if err != nil {
			logrus.WithError(err).Fatal("Error creating metric(s).")
		}

		err = sendSpan(nconn, span)
		if err != nil {
			logrus.WithError(err).Fatal("Error sending metric(s).")
		}
	} else {
		var conn MinimalClient
		conn, err = statsd.New(addr)
		if err != nil {
			logrus.WithError(err).Fatal("Error!")
		}

		tags := tags(*tag)
		err = sendMetrics(conn, passedFlags, *name, tags)
		if err != nil {
			logrus.WithError(err).Fatal("Error sending metric(s).")
		}
	}
}

func flags() map[string]bool {
	flag.Parse()
	// hacky way to detect which flags were *actually* set
	passedFlags := make(map[string]bool)
	flag.Visit(func(f *flag.Flag) {
		passedFlags[f.Name] = true
	})
	return passedFlags
}

func tags(tag string) []string {
	var tags []string
	for _, elem := range strings.Split(tag, ",") {
		tags = append(tags, elem)
	}
	return tags
}

func addr(passedFlags map[string]bool, conf *veneur.Config, hostport *string) (string, error) {
	addr := ""
	var err error
	if conf != nil {
		addr = conf.UdpAddress // TODO: conf.TcpAddress
	} else if passedFlags["hostport"] && hostport != nil {
		addr = *hostport
	} else {
		err = errors.New("you must either specify a Veneur config file or a valid hostport")
	}
	return addr, err
}

func bareMetric(name string, tags string) *ssf.SSFSample {
	metric := &ssf.SSFSample{}
	metric.Name = name
	metric.Tags = make(map[string]string)
	for _, elem := range strings.Split(tags, ",") {
		tag := strings.Split(elem, ":")
		metric.Tags[tag[0]] = tag[1]
	}
	return metric
}

func createMetrics(passedFlags map[string]bool, name string, tags string) (*ssf.SSFSpan, error) {
	var err error
	span := &ssf.SSFSpan{}
	if passedFlags["gauge"] {
		logrus.Debugf("Sending gauge '%s' -> %f", name, *gauge)
		metric := bareMetric(name, tags)
		metric.Metric = ssf.SSFSample_GAUGE
		metric.Value = float32(*gauge)
		span.Metrics = append(span.Metrics, metric)
	}
	// TODO: figure out timing metrics
	if passedFlags["count"] {
		logrus.Debugf("Sending count '%s' -> %d", name, *count)
		metric := bareMetric(name, tags)
		metric.Metric = ssf.SSFSample_COUNTER
		metric.Value = float32(*count)
		span.Metrics = append(span.Metrics, metric)
	}
	return span, err
}

func sendSpan(conn MinimalConn, span *ssf.SSFSpan) error {
	var err error
	var data []byte
	data, err = proto.Marshal(span)
	if err != nil {
		return err
	}

	_, err = conn.Write(data)
	if err != nil {
		return err
	}
	return nil
}

func sendMetrics(client MinimalClient, passedFlags map[string]bool, name string, tags []string) error {
	var err error
	if passedFlags["gauge"] {
		logrus.Debugf("Sending gauge '%s' -> %f", name, *gauge)
		err = client.Gauge(name, *gauge, tags, 1)
		if err != nil {
			return err
		}
	}
	if passedFlags["timing"] {
		logrus.Debugf("Sending timing '%s' -> %f", name, *timing)
		err = client.Timing(name, *timing, tags, 1)
		if err != nil {
			return err
		}
	}
	if passedFlags["count"] {
		logrus.Debugf("Sending count '%s' -> %d", name, *count)
		err = client.Count(name, *count, tags, 1)
		if err != nil {
			return err
		}
	}
	if !passedFlags["gauge"] && !passedFlags["timing"] && !passedFlags["timeinms"] && !passedFlags["count"] {
		logrus.Info("No metrics reported.")
	}
	return err
	// TODO: error on no metrics?
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"text/tabwriter"

	"github.com/dlespiau/x86db"
)

func nasmOpcodeToPlan9(op string) string {
	/*
	 * SSE
	 */
	switch op {
	// The condition is encoded as an imm8 operand of CMPPS
	case "CMPEQPS", "CMPLTPS", "CMPLEPS", "CMPUNORDPS", "CMPNEQPS", "CMPNLTPS",
		"CMPNLEPS", "CMPORDPS":
		return "CMPPS"
	// The condition is encoded as an imm8 operand of CMPSS.
	case "CMPEQSS", "CMPLTSS", "CMPLESS", "CMPUNORDSS", "CMPNEQSS", "CMPNLTSS",
		"CMPNLESS", "CMPORDSS":
		return "CMPSS"
	case "CVTSI2SS":
		return "CVTSL2SS"
	case "CVTSS2SI":
		return "CVTSS2SL"
	case "CVTTSS2SI":
		return "CVTTSS2SL"
	}

	/*
	 * PCLMULQDQ
	 */
	switch op {
	case "PCLMULLQLQDQ", "PCLMULHQLQDQ", "PCLMULLQHQDQ", "PCLMULHQHQDQ":
		return "PCLMULQDQ"
	}

	return op
}

func isAlreadyKnown(name string) bool {
	name = nasmOpcodeToPlan9(name)
	for _, opcode := range Anames {
		if name == opcode {
			return true
		}
	}
	return false
}

func isAlreadyTested(name string) bool {
	name = nasmOpcodeToPlan9(name)
	_, ok := testedMap[name]
	return ok
}

func main() {
	db := x86db.NewDB(os.Args[1])

	err := db.Open()
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	// SSE2 instructions, not already known by the Go's cmd/internal/obj package.
	insns := db.FindByExtension(x86db.ExtensionSSE2).Where(func(insn x86db.Instruction) bool {
		return !isAlreadyKnown(insn.Name)
	})

	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
	for _, insn := range insns {
		fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", insn.Name, insn.Operands, insn.Pattern, insn.Flags)
	}
	w.Flush()
}
<commit_msg>gogen: CMPPD and CPMSD encode the condition as an immediate operand<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"text/tabwriter"

	"github.com/dlespiau/x86db"
)

func nasmOpcodeToPlan9(op string) string {
	/*
	 * SSE
	 */
	switch op {
	// The condition is encoded as an imm8 operand of CMPPS
	case "CMPEQPS", "CMPLTPS", "CMPLEPS", "CMPUNORDPS", "CMPNEQPS", "CMPNLTPS",
		"CMPNLEPS", "CMPORDPS":
		return "CMPPS"
	// The condition is encoded as an imm8 operand of CMPSS.
	case "CMPEQSS", "CMPLTSS", "CMPLESS", "CMPUNORDSS", "CMPNEQSS", "CMPNLTSS",
		"CMPNLESS", "CMPORDSS":
		return "CMPSS"
	case "CVTSI2SS":
		return "CVTSL2SS"
	case "CVTSS2SI":
		return "CVTSS2SL"
	case "CVTTSS2SI":
		return "CVTTSS2SL"
	}

	/*
	 * PCLMULQDQ
	 */
	switch op {
	case "PCLMULLQLQDQ", "PCLMULHQLQDQ", "PCLMULLQHQDQ", "PCLMULHQHQDQ":
		return "PCLMULQDQ"
	}

	/*
	 * SSE2
	 */
	switch op {
	// The condition is encoded as an imm8 operand of CMPPD
	case "CMPEQPD", "CMPLTPD", "CMPLEPD", "CMPUNORDPD", "CMPNEQPD", "CMPNLTPD",
		"CMPNLEPD", "CMPORDPD":
		return "CMPPD"
	// The condition is encoded as an imm8 operand of CMPSD.
	case "CMPEQSD", "CMPLTSD", "CMPLESD", "CMPUNORDSD", "CMPNEQSD", "CMPNLTSD",
		"CMPNLESD", "CMPORDSD":
		return "CMPSD"
	}

	return op
}

func isAlreadyKnown(name string) bool {
	name = nasmOpcodeToPlan9(name)
	for _, opcode := range Anames {
		if name == opcode {
			return true
		}
	}
	return false
}

func isAlreadyTested(name string) bool {
	name = nasmOpcodeToPlan9(name)
	_, ok := testedMap[name]
	return ok
}

func main() {
	db := x86db.NewDB(os.Args[1])

	err := db.Open()
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	// SSE2 instructions, not already known by the Go's cmd/internal/obj package.
	insns := db.FindByExtension(x86db.ExtensionSSE2).Where(func(insn x86db.Instruction) bool {
		return !isAlreadyKnown(insn.Name)
	})

	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
	for _, insn := range insns {
		fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", insn.Name, insn.Operands, insn.Pattern, insn.Flags)
	}
	w.Flush()
}
<|endoftext|>


<commit_before>//go:build integration

package cmd

import (
	"fmt"
	"github.com/qiniu/qshell/v2/cmd_test/test"
	"path/filepath"
	"strings"
	"testing"
)

func TestQUpload(t *testing.T) {
	deleteFile(t, "1K.tmp")
	deleteFile(t, "32K.tmp")
	deleteFile(t, "64K.tmp")
	deleteFile(t, "256K.tmp")
	copyFile(t, test.Key, "512K.tmp")
	copyFile(t, test.Key, "1024K.tmp")

	fileSizeList := []int{1, 32, 64, 256, 512, 1024, 2 * 1024, 4 * 1024, 5 * 1024, 8 * 1024, 10 * 1024}
	fileSizeList = []int{1, 2}
	for _, size := range fileSizeList {
		test.CreateTempFile(size)
	}

	fileDir, err := test.TempPath()
	if err != nil {
		t.Fatal("create upload temp file error:", err)
	}

	resultPath, err := test.ResultPath()
	if err != nil {
		t.Fatal("get result path error:", err)
	}

	successLogPath := filepath.Join(resultPath, "qupload2_success.txt")
	failLogPath := filepath.Join(resultPath, "qupload2_fail.txt")
	overwriteLogPath := filepath.Join(resultPath, "qupload2_overwrite.txt")
	logPath := filepath.Join(resultPath, "qupload2_log.txt")
	recordPath := filepath.Join(resultPath, "record")

	cfgContent := fmt.Sprintf(`{
	"bucket": "%s",
	"src_dir": "%s",
	"log_stdout": true,
	"overwrite": true,
	"check_exists": true,
	"check_size": true,
	"work_count": 4,
	"log_file": "%s",
	"record_root": "%s"
}`, test.Bucket, fileDir, logPath, recordPath)
	cfgFile, err := test.CreateFileWithContent("upload_cfg.json", cfgContent)
	defer test.RemoveFile(cfgFile)

	if err != nil {
		t.Fatal("create cdn config file error:", err)
	}

	test.RunCmdWithError("qupload", cfgFile,
		"--success-list", successLogPath,
		"--failure-list", failLogPath,
		"--overwrite-list", overwriteLogPath)
	defer func() {
		test.RemoveFile(successLogPath)
		test.RemoveFile(failLogPath)
		test.RemoveFile(overwriteLogPath)
		test.RemoveFile(logPath)
		test.RemoveFile(recordPath)
	}()

	if !test.IsFileHasContent(successLogPath) {
		t.Fatal("batch result: success log to file error: file empty")
	}

	if test.IsFileHasContent(failLogPath) {
		t.Fatal("batch result: fail log to file error: file should empty")
	}

	if !test.IsFileHasContent(overwriteLogPath) {
		t.Fatal("batch result: overwrite log to file error: file empty")
	}

	if !test.IsFileHasContent(logPath) {
		t.Fatal("batch result: log to file error: file empty")
	}
}

func TestQUploadDocument(t *testing.T) {
	test.TestDocument("qupload", t)
}

func TestQUpload2WithSrcDir(t *testing.T) {
	deleteFile(t, "1K.tmp")
	deleteFile(t, "32K.tmp")
	deleteFile(t, "64K.tmp")
	deleteFile(t, "256K.tmp")

	fileSizeList := []int{1, 32, 64, 256, 512, 1024, 2 * 1024, 4 * 1024, 5 * 1024, 8 * 1024, 10 * 1024}
	for _, size := range fileSizeList {
		test.CreateTempFile(size)
	}

	fileDir, err := test.TempPath()
	if err != nil {
		t.Fatal("create upload temp file error:", err)
	}

	resultPath, err := test.ResultPath()
	if err != nil {
		t.Fatal("get result path error:", err)
	}

	fileListPath := filepath.Join(resultPath, "qupload2_file_list.txt")
	_, errs := test.RunCmdWithError("dircache", fileDir,
		"-o", fileListPath)
	if len(errs) > 0 {
		t.Fatal("upload2 dircache error:", err)
	}

	result, errs := test.RunCmdWithError("qupload2",
		"--bucket", test.Bucket,
		"--src-dir", fileDir,
		"--rescan-local",
		"--check-exists")
	defer func() {
		test.RemoveFile(fileListPath)
	}()

	if len(errs) > 0 {
		t.Fail()
	}

	result = strings.ReplaceAll(result, "\n", "")
	if !strings.Contains(result, "Upload File success") {
		t.Fatal(result)
	}
}

func TestQUpload2WithFileList(t *testing.T) {
	deleteFile(t, "1K.tmp")
	deleteFile(t, "32K.tmp")
	deleteFile(t, "64K.tmp")
	deleteFile(t, "256K.tmp")
	copyFile(t, test.Key, "512K.tmp")
	copyFile(t, test.Key, "1024K.tmp")
	fileSizeList := []int{1, 32, 64, 256, 512, 1024, 2 * 1024, 4 * 1024, 5 * 1024, 8 * 1024, 10 * 1024}
	for _, size := range fileSizeList {
		test.CreateTempFile(size)
	}

	fileDir, err := test.TempPath()
	if err != nil {
		t.Fatal("create upload temp file error:", err)
	}

	resultPath, err := test.ResultPath()
	if err != nil {
		t.Fatal("get result path error:", err)
	}

	successLogPath := filepath.Join(resultPath, "qupload2_success.txt")
	failLogPath := filepath.Join(resultPath, "qupload2_fail.txt")
	overwriteLogPath := filepath.Join(resultPath, "qupload2_overwrite.txt")
	logPath := filepath.Join(resultPath, "qupload2_log.txt")
	recordPath := filepath.Join(resultPath, "record")

	fileListPath := filepath.Join(resultPath, "qupload2_file_list.txt")
	_, errs := test.RunCmdWithError("dircache", fileDir,
		"-o", fileListPath)
	if len(errs) > 0 {
		t.Fatal("upload2 dircache error:", err)
	}

	if err := test.AppendToFile(fileListPath, `
mock01.jpg	10485760	16455233472998522
mock02.jpg	10485760	16455233472998522
`); err != nil {
		t.Fatal("upload2 upload file list append error:", err)
	}

	test.RunCmdWithError("qupload2",
		"--bucket", test.Bucket,
		"--src-dir", fileDir,
		"--file-list", fileListPath,
		"--overwrite",
		"--check-exists",
		"--check-hash",
		"--check-size",
		"--file-type", "1",
		"--rescan-local", "false",
		"--ignore-dir", "",
		"--key-prefix", "",
		"--skip-file-prefixes", "",
		"--skip-fixed-strings", "",
		"--skip-path-prefixes", "",
		"--skip-suffixes", "",
		"--thread-count", "4",
		"--success-list", successLogPath,
		"--failure-list", failLogPath,
		"--overwrite-list", overwriteLogPath,
		"--record-root", recordPath,
		"--log-file", logPath,
		"--log-level", "debug",
		"--log-rotate", "10",
		"--up-host", "",
		"-d")

	defer func() {
		test.RemoveFile(successLogPath)
		test.RemoveFile(failLogPath)
		test.RemoveFile(overwriteLogPath)
		test.RemoveFile(logPath)
		test.RemoveFile(recordPath)
	}()

	if !test.IsFileHasContent(successLogPath) {
		t.Fatal("batch result: success log to file error: file empty")
	}

	if !test.IsFileHasContent(failLogPath) {
		t.Fatal("batch result: fail log  to file error: file empty")
	}

	if !test.IsFileHasContent(overwriteLogPath) {
		t.Fatal("batch result: overwrite log to file error: file empty")
	}

	if !test.IsFileHasContent(logPath) {
		t.Fatal("batch result: log to file error: file empty")
	}
}

func TestQUpload2NoBucket(t *testing.T) {
	_, errs := test.RunCmdWithError("qupload2")
	if !strings.Contains(errs, "Bucket can't empty") {
		t.Fatal(errs)
	}
}

func TestQUpload2NoSrc(t *testing.T) {
	_, errs := test.RunCmdWithError("qupload2",
		"--bucket", test.Bucket)
	if !strings.Contains(errs, "SrcDir can't empty") {
		t.Fatal(errs)
	}
}

func TestQUpload2NotExistSrcDir(t *testing.T) {
	_, errs := test.RunCmdWithError("qupload2",
		"--bucket", test.Bucket,
		"--src-dir", "/Demo")
	if !strings.Contains(errs, "no such file or directory") {
		t.Fatal(errs)
	}
}

func TestQUpload2Document(t *testing.T) {
	test.TestDocument("qupload2", t)
}
<commit_msg>upload test case add key_prefix<commit_after>//go:build integration

package cmd

import (
	"fmt"
	"github.com/qiniu/qshell/v2/cmd_test/test"
	"path/filepath"
	"strings"
	"testing"
)

func TestQUpload(t *testing.T) {
	deleteFile(t, "1K.tmp")
	deleteFile(t, "32K.tmp")
	deleteFile(t, "64K.tmp")
	deleteFile(t, "256K.tmp")
	copyFile(t, test.Key, "512K.tmp")
	copyFile(t, test.Key, "1024K.tmp")

	fileSizeList := []int{1, 32, 64, 256, 512, 1024, 2 * 1024, 4 * 1024, 5 * 1024, 8 * 1024, 10 * 1024}
	fileSizeList = []int{1, 2}
	for _, size := range fileSizeList {
		test.CreateTempFile(size)
	}

	fileDir, err := test.TempPath()
	if err != nil {
		t.Fatal("create upload temp file error:", err)
	}

	resultPath, err := test.ResultPath()
	if err != nil {
		t.Fatal("get result path error:", err)
	}

	successLogPath := filepath.Join(resultPath, "qupload2_success.txt")
	failLogPath := filepath.Join(resultPath, "qupload2_fail.txt")
	overwriteLogPath := filepath.Join(resultPath, "qupload2_overwrite.txt")
	logPath := filepath.Join(resultPath, "qupload2_log.txt")
	recordPath := filepath.Join(resultPath, "record")

	cfgContent := fmt.Sprintf(`{
	"bucket": "%s",
	"src_dir": "%s",
	"log_stdout": true,
	"overwrite": true,
	"check_exists": true,
	"check_size": true,
	"work_count": 4,
	"key_prefix": "test/",
	"log_file": "%s",
	"record_root": "%s"
}`, test.Bucket, fileDir, logPath, recordPath)
	cfgFile, err := test.CreateFileWithContent("upload_cfg.json", cfgContent)
	defer test.RemoveFile(cfgFile)

	if err != nil {
		t.Fatal("create cdn config file error:", err)
	}

	test.RunCmdWithError("qupload", cfgFile,
		"--success-list", successLogPath,
		"--failure-list", failLogPath,
		"--overwrite-list", overwriteLogPath)
	defer func() {
		test.RemoveFile(successLogPath)
		test.RemoveFile(failLogPath)
		test.RemoveFile(overwriteLogPath)
		test.RemoveFile(logPath)
		test.RemoveFile(recordPath)
	}()

	if !test.IsFileHasContent(successLogPath) {
		t.Fatal("batch result: success log to file error: file empty")
	}

	if test.IsFileHasContent(failLogPath) {
		t.Fatal("batch result: fail log to file error: file should empty")
	}

	if !test.IsFileHasContent(overwriteLogPath) {
		t.Fatal("batch result: overwrite log to file error: file empty")
	}

	if !test.IsFileHasContent(logPath) {
		t.Fatal("batch result: log to file error: file empty")
	}
}

func TestQUploadDocument(t *testing.T) {
	test.TestDocument("qupload", t)
}

func TestQUpload2WithSrcDir(t *testing.T) {
	deleteFile(t, "1K.tmp")
	deleteFile(t, "32K.tmp")
	deleteFile(t, "64K.tmp")
	deleteFile(t, "256K.tmp")

	fileSizeList := []int{1, 32, 64, 256, 512, 1024, 2 * 1024, 4 * 1024, 5 * 1024, 8 * 1024, 10 * 1024}
	for _, size := range fileSizeList {
		test.CreateTempFile(size)
	}

	fileDir, err := test.TempPath()
	if err != nil {
		t.Fatal("create upload temp file error:", err)
	}

	resultPath, err := test.ResultPath()
	if err != nil {
		t.Fatal("get result path error:", err)
	}

	fileListPath := filepath.Join(resultPath, "qupload2_file_list.txt")
	_, errs := test.RunCmdWithError("dircache", fileDir,
		"-o", fileListPath)
	if len(errs) > 0 {
		t.Fatal("upload2 dircache error:", err)
	}

	result, errs := test.RunCmdWithError("qupload2",
		"--bucket", test.Bucket,
		"--src-dir", fileDir,
		"--rescan-local",
		"--check-exists")
	defer func() {
		test.RemoveFile(fileListPath)
	}()

	if len(errs) > 0 {
		t.Fail()
	}

	result = strings.ReplaceAll(result, "\n", "")
	if !strings.Contains(result, "Upload File success") {
		t.Fatal(result)
	}
}

func TestQUpload2WithFileList(t *testing.T) {
	deleteFile(t, "1K.tmp")
	deleteFile(t, "32K.tmp")
	deleteFile(t, "64K.tmp")
	deleteFile(t, "256K.tmp")
	copyFile(t, test.Key, "512K.tmp")
	copyFile(t, test.Key, "1024K.tmp")
	fileSizeList := []int{1, 32, 64, 256, 512, 1024, 2 * 1024, 4 * 1024, 5 * 1024, 8 * 1024, 10 * 1024}
	for _, size := range fileSizeList {
		test.CreateTempFile(size)
	}

	fileDir, err := test.TempPath()
	if err != nil {
		t.Fatal("create upload temp file error:", err)
	}

	resultPath, err := test.ResultPath()
	if err != nil {
		t.Fatal("get result path error:", err)
	}

	successLogPath := filepath.Join(resultPath, "qupload2_success.txt")
	failLogPath := filepath.Join(resultPath, "qupload2_fail.txt")
	overwriteLogPath := filepath.Join(resultPath, "qupload2_overwrite.txt")
	logPath := filepath.Join(resultPath, "qupload2_log.txt")
	recordPath := filepath.Join(resultPath, "record")

	fileListPath := filepath.Join(resultPath, "qupload2_file_list.txt")
	_, errs := test.RunCmdWithError("dircache", fileDir,
		"-o", fileListPath)
	if len(errs) > 0 {
		t.Fatal("upload2 dircache error:", err)
	}

	if err := test.AppendToFile(fileListPath, `
mock01.jpg	10485760	16455233472998522
mock02.jpg	10485760	16455233472998522
`); err != nil {
		t.Fatal("upload2 upload file list append error:", err)
	}

	test.RunCmdWithError("qupload2",
		"--bucket", test.Bucket,
		"--src-dir", fileDir,
		"--file-list", fileListPath,
		"--overwrite",
		"--check-exists",
		"--check-hash",
		"--check-size",
		"--file-type", "1",
		"--rescan-local", "false",
		"--ignore-dir", "",
		"--key-prefix", "",
		"--skip-file-prefixes", "",
		"--skip-fixed-strings", "",
		"--skip-path-prefixes", "",
		"--skip-suffixes", "",
		"--thread-count", "4",
		"--success-list", successLogPath,
		"--failure-list", failLogPath,
		"--overwrite-list", overwriteLogPath,
		"--record-root", recordPath,
		"--log-file", logPath,
		"--log-level", "debug",
		"--log-rotate", "10",
		"--up-host", "",
		"-d")

	defer func() {
		test.RemoveFile(successLogPath)
		test.RemoveFile(failLogPath)
		test.RemoveFile(overwriteLogPath)
		test.RemoveFile(logPath)
		test.RemoveFile(recordPath)
	}()

	if !test.IsFileHasContent(successLogPath) {
		t.Fatal("batch result: success log to file error: file empty")
	}

	if !test.IsFileHasContent(failLogPath) {
		t.Fatal("batch result: fail log  to file error: file empty")
	}

	if !test.IsFileHasContent(overwriteLogPath) {
		t.Fatal("batch result: overwrite log to file error: file empty")
	}

	if !test.IsFileHasContent(logPath) {
		t.Fatal("batch result: log to file error: file empty")
	}
}

func TestQUpload2NoBucket(t *testing.T) {
	_, errs := test.RunCmdWithError("qupload2")
	if !strings.Contains(errs, "Bucket can't empty") {
		t.Fatal(errs)
	}
}

func TestQUpload2NoSrc(t *testing.T) {
	_, errs := test.RunCmdWithError("qupload2",
		"--bucket", test.Bucket)
	if !strings.Contains(errs, "SrcDir can't empty") {
		t.Fatal(errs)
	}
}

func TestQUpload2NotExistSrcDir(t *testing.T) {
	_, errs := test.RunCmdWithError("qupload2",
		"--bucket", test.Bucket,
		"--src-dir", "/Demo")
	if !strings.Contains(errs, "no such file or directory") {
		t.Fatal(errs)
	}
}

func TestQUpload2Document(t *testing.T) {
	test.TestDocument("qupload2", t)
}
<|endoftext|>


<commit_before>package application

import (
	"context"
	_ "expvar" // Register the expvar handlers
	"net/http"
	_ "net/http/pprof" // Register the pprof handlers
)

// DebugAdapter ./...
type DebugAdapter struct {
	address string
	*http.ServeMux
}

// NewDebugAdapter provides new debug adapter
// /debug/pprof - Added to the default mux by importing the net/http/pprof package.
// /debug/vars - Added to the default mux by importing the expvar package.
func NewDebugAdapter(address string) *DebugAdapter {
	return &DebugAdapter{
		address:  address,
		ServeMux: http.DefaultServeMux,
	}
}

// Start starts http server
func (adapter *DebugAdapter) Start(ctx context.Context) error {
	return http.ListenAndServe(adapter.address, adapter)
}

// Stop stops debug server
// does nothing always returns nil
func (adapter *DebugAdapter) Stop(ctx context.Context) error {
	return nil
}
<commit_msg>Update debug adapter<commit_after>package application

import (
	"context"
	_ "expvar" // Register the expvar handlers
	"net/http"
	_ "net/http/pprof" // Register the pprof handlers
)

// DebugAdapter ./...
type DebugAdapter struct {
	*http.Server
}

// NewDebugAdapter provides new debug adapter
// /debug/pprof - Added to the default mux by importing the net/http/pprof package.
// /debug/vars - Added to the default mux by importing the expvar package.
func NewDebugAdapter(address string) *DebugAdapter {
	return &DebugAdapter{
		&http.Server{
			Addr:    address,
			Handler: http.DefaultServeMux,
		},
	}
}

// Start start http application adapter
func (adapter *DebugAdapter) Start(ctx context.Context) error {
	return adapter.ListenAndServe()
}

// Stop stops http application adapter
func (adapter *DebugAdapter) Stop(ctx context.Context) error {
	return adapter.Shutdown(ctx)
}
<|endoftext|>