<commit_before>package server

import (
	"log"
	"os"
	"sync/atomic"
	"time"

	"github.com/privacylab/talek/common"
)

// Frontend terminates client connections to the leader server.
// It is the point of global serialization, and establishes sequence numbers.
type Frontend struct {
	// Private State
	log  *log.Logger
	name string
	*Config

	proposedSeqNo uint64 // Use atomic.AddUint64, atomic.LoadUint64
	readChan      chan *readRequest

	replicas []common.ReplicaInterface
	dead     int32
}

// readRequest is the grouped request and reply memory used for batching
// incoming reads onto a single thread.
type readRequest struct {
	Args  *common.EncodedReadArgs
	Reply *common.ReadReply
	Done  chan bool
}

// NewFrontend creates a new Frontend for a provided configuration.
func NewFrontend(name string, config *Config, replicas []common.ReplicaInterface) *Frontend {
	fe := &Frontend{}
	fe.log = log.New(os.Stdout, "[Frontend:"+name+"] ", log.Ldate|log.Ltime|log.Lshortfile)
	fe.name = name
	fe.Config = config
	fe.replicas = replicas
	fe.readChan = make(chan *readRequest, 10)

	// Periodically serialize database epoch advances.
	go fe.periodicWrite()
	// Batch incoming reads into combined requests to replicas.
	go fe.batchReads()

	return fe
}

/** PUBLIC METHODS (threadsafe) **/

// Close goroutines associated with this object.
func (fe *Frontend) Close() {
	atomic.StoreInt32(&fe.dead, 1)
}

// GetName exports the name of the server.
func (fe *Frontend) GetName(args *interface{}, reply *string) error {
	*reply = fe.name
	return nil
}

// GetConfig returns the current common configuration from the server.
func (fe *Frontend) GetConfig(args *interface{}, reply *common.Config) error {
	config := *fe.Config.Config
	*reply = config
	return nil
}

func (fe *Frontend) Write(args *common.WriteArgs, reply *common.WriteReply) error {
	seqNo := atomic.AddUint64(&fe.proposedSeqNo, 1)
	args.GlobalSeqNo = seqNo

	replicaWrite := &common.ReplicaWriteArgs{
		WriteArgs: *args,
	}
	replicaReply := common.ReplicaWriteReply{}
	//@todo writes in parallel
	for i, r := range fe.replicas {
		err := r.Write(replicaWrite, &replicaReply)
		if err != nil {
			reply.Err = err.Error()
			fe.log.Fatalf("Error writing to replica %d: %v", i, err)
		} else if len(replicaReply.Err) > 0 {
			reply.Err = replicaReply.Err
		}
	}

	return nil
}

func (fe *Frontend) Read(args *common.EncodedReadArgs, reply *common.ReadReply) error {
	ready := make(chan bool, 1)
	fe.readChan <- &readRequest{Args: args, Reply: reply, Done: ready}
	<-ready

	return nil
}

// GetUpdates provides the most recent global interest vector deltas.
func (fe *Frontend) GetUpdates(args *common.GetUpdatesArgs, reply *common.GetUpdatesReply) error {
	fe.log.Println("GetUpdates: ")
	// @TODO
	return nil
}

// periodicWrite runs until the dead flag is set, and periodically send a write
// request to all replicas telling them to advance their write epoch.
func (fe *Frontend) periodicWrite() {
	for atomic.LoadInt32(&fe.dead) == 0 {
		tick := time.After(fe.WriteInterval)
		select {
		case <-tick:
			args := &common.ReplicaWriteArgs{
				EpochFlag: true,
			}
			var rep common.ReplicaWriteReply
			for _, r := range fe.replicas {
				r.Write(args, &rep)
			}
		}
	}
}

func (fe *Frontend) batchReads() {
	batch := make([]*readRequest, 0, fe.Config.ReadBatch)
	var readReq *readRequest
	tick := time.After(fe.Config.ReadInterval)
	for atomic.LoadInt32(&fe.dead) == 0 {
		select {
		case readReq = <-fe.readChan:
			batch = append(batch, readReq)
			if len(batch) >= fe.Config.ReadBatch {
				go fe.triggerBatchRead(batch)
				batch = make([]*readRequest, 0, fe.Config.ReadBatch)
			} else {
				fe.log.Printf("Read: add to batch, size=%v\n", len(batch))
			}
			continue
		case <-tick:
			if len(batch) > 0 {
				go fe.triggerBatchRead(batch)
				batch = make([]*readRequest, 0, fe.Config.ReadBatch)
			}
			tick = time.After(fe.Config.ReadInterval)
			continue
		}
	}
}

func (fe *Frontend) triggerBatchRead(batch []*readRequest) error {
	args := &common.BatchReadRequest{}
	// Copy args
	args.Args = make([]common.EncodedReadArgs, len(batch), len(batch))
	for i, val := range batch {
		if val.Args != nil {
			args.Args[i] = *val.Args
		}
	}

	// Choose a SeqNoRange
	currSeqNo := atomic.LoadUint64(&fe.proposedSeqNo) + 1
	if currSeqNo <= uint64(fe.Config.WindowSize()) {
		args.SeqNoRange.Start = 1 // Minimum of 1
	} else {
		args.SeqNoRange.Start = currSeqNo - uint64(fe.Config.WindowSize()) // Inclusive
	}
	args.SeqNoRange.End = currSeqNo // Exclusive
	args.SeqNoRange.Aborted = make([]uint64, 0, 0)

	// Start computation
	// @todo reads in parallel
	replies := make([]common.BatchReadReply, len(fe.replicas))
	for i, r := range fe.replicas {
		err := r.BatchRead(args, &replies[i])
		if err != nil || replies[i].Err != "" {
			fe.log.Fatalf("Error making read to replica %d: %v%v", i, err, replies[i].Err)
		}
		if len(replies[i].Replies) != len(batch) {
			fe.log.Fatalf("Replica %d gave the wrong number of replies (%d instead of %d)", i, len(replies[i].Replies), len(batch))
		}
	}

	// Respond to clients
	// @todo propagate errors back to clients.
	for i, val := range batch {
		for _, rp := range replies {
			val.Reply.Combine(rp.Replies[i].Data)
		}
		val.Reply.GlobalSeqNo = args.SeqNoRange
		val.Done <- true
	}

	return nil
}
<commit_msg>allocate bytes for frontend replies<commit_after>package server

import (
	"errors"
	"log"
	"os"
	"sync/atomic"
	"time"

	"github.com/privacylab/talek/common"
)

// Frontend terminates client connections to the leader server.
// It is the point of global serialization, and establishes sequence numbers.
type Frontend struct {
	// Private State
	log  *log.Logger
	name string
	*Config

	proposedSeqNo uint64 // Use atomic.AddUint64, atomic.LoadUint64
	readChan      chan *readRequest

	replicas []common.ReplicaInterface
	dead     int32
}

// readRequest is the grouped request and reply memory used for batching
// incoming reads onto a single thread.
type readRequest struct {
	Args  *common.EncodedReadArgs
	Reply *common.ReadReply
	Done  chan bool
}

// NewFrontend creates a new Frontend for a provided configuration.
func NewFrontend(name string, config *Config, replicas []common.ReplicaInterface) *Frontend {
	fe := &Frontend{}
	fe.log = log.New(os.Stdout, "[Frontend:"+name+"] ", log.Ldate|log.Ltime|log.Lshortfile)
	fe.name = name
	fe.Config = config
	fe.replicas = replicas
	fe.readChan = make(chan *readRequest, 10)

	// Periodically serialize database epoch advances.
	go fe.periodicWrite()
	// Batch incoming reads into combined requests to replicas.
	go fe.batchReads()

	return fe
}

/** PUBLIC METHODS (threadsafe) **/

// Close goroutines associated with this object.
func (fe *Frontend) Close() {
	atomic.StoreInt32(&fe.dead, 1)
}

// GetName exports the name of the server.
func (fe *Frontend) GetName(args *interface{}, reply *string) error {
	*reply = fe.name
	return nil
}

// GetConfig returns the current common configuration from the server.
func (fe *Frontend) GetConfig(args *interface{}, reply *common.Config) error {
	config := *fe.Config.Config
	*reply = config
	return nil
}

func (fe *Frontend) Write(args *common.WriteArgs, reply *common.WriteReply) error {
	seqNo := atomic.AddUint64(&fe.proposedSeqNo, 1)
	args.GlobalSeqNo = seqNo

	replicaWrite := &common.ReplicaWriteArgs{
		WriteArgs: *args,
	}
	replicaReply := common.ReplicaWriteReply{}
	//@todo writes in parallel
	for i, r := range fe.replicas {
		err := r.Write(replicaWrite, &replicaReply)
		if err != nil {
			reply.Err = err.Error()
			fe.log.Fatalf("Error writing to replica %d: %v", i, err)
		} else if len(replicaReply.Err) > 0 {
			reply.Err = replicaReply.Err
		}
	}

	return nil
}

func (fe *Frontend) Read(args *common.EncodedReadArgs, reply *common.ReadReply) error {
	ready := make(chan bool, 1)
	fe.readChan <- &readRequest{Args: args, Reply: reply, Done: ready}
	<-ready

	return nil
}

// GetUpdates provides the most recent global interest vector deltas.
func (fe *Frontend) GetUpdates(args *common.GetUpdatesArgs, reply *common.GetUpdatesReply) error {
	fe.log.Println("GetUpdates: ")
	// @TODO
	return nil
}

// periodicWrite runs until the dead flag is set, and periodically send a write
// request to all replicas telling them to advance their write epoch.
func (fe *Frontend) periodicWrite() {
	for atomic.LoadInt32(&fe.dead) == 0 {
		tick := time.After(fe.WriteInterval)
		select {
		case <-tick:
			args := &common.ReplicaWriteArgs{
				EpochFlag: true,
			}
			var rep common.ReplicaWriteReply
			for _, r := range fe.replicas {
				r.Write(args, &rep)
			}
		}
	}
}

func (fe *Frontend) batchReads() {
	batch := make([]*readRequest, 0, fe.Config.ReadBatch)
	var readReq *readRequest
	tick := time.After(fe.Config.ReadInterval)
	for atomic.LoadInt32(&fe.dead) == 0 {
		select {
		case readReq = <-fe.readChan:
			batch = append(batch, readReq)
			if len(batch) >= fe.Config.ReadBatch {
				go fe.triggerBatchRead(batch)
				batch = make([]*readRequest, 0, fe.Config.ReadBatch)
			} else {
				fe.log.Printf("Read: add to batch, size=%v\n", len(batch))
			}
			continue
		case <-tick:
			if len(batch) > 0 {
				go fe.triggerBatchRead(batch)
				batch = make([]*readRequest, 0, fe.Config.ReadBatch)
			}
			tick = time.After(fe.Config.ReadInterval)
			continue
		}
	}
}

func (fe *Frontend) triggerBatchRead(batch []*readRequest) error {
	args := &common.BatchReadRequest{}
	// Copy args
	args.Args = make([]common.EncodedReadArgs, len(batch), len(batch))
	for i, val := range batch {
		if val.Args != nil {
			args.Args[i] = *val.Args
		}
	}

	// Choose a SeqNoRange
	currSeqNo := atomic.LoadUint64(&fe.proposedSeqNo) + 1
	if currSeqNo <= uint64(fe.Config.WindowSize()) {
		args.SeqNoRange.Start = 1 // Minimum of 1
	} else {
		args.SeqNoRange.Start = currSeqNo - uint64(fe.Config.WindowSize()) // Inclusive
	}
	args.SeqNoRange.End = currSeqNo // Exclusive
	args.SeqNoRange.Aborted = make([]uint64, 0, 0)

	// Start computation
	// @todo reads in parallel
	var replicaErr error
	replies := make([]common.BatchReadReply, len(fe.replicas))
	for i, r := range fe.replicas {
		err := r.BatchRead(args, &replies[i])
		if err != nil || replies[i].Err != "" {
			replicaErr = err
			fe.log.Fatalf("Error making read to replica %d: %v%v", i, err, replies[i].Err)
		}
		if len(replies[i].Replies) != len(batch) {
			replicaErr = errors.New("failure from Replica " + i)
			fe.log.Fatalf("Replica %d gave the wrong number of replies (%d instead of %d)", i, len(replies[i].Replies), len(batch))
		}
	}

	// Respond to clients
	// @todo propagate errors back to clients.
	replyLength := len(replies[0].Replies[0].Data)
	for i, val := range batch {
		val.Reply.Data = make([]byte, replyLength)
		for _, rp := range replies {
			val.Reply.Combine(rp.Replies[i].Data)
		}
		if replicaErr != nil {
			val.Reply.Err = replicaErr.Error()
		}
		val.Reply.GlobalSeqNo = args.SeqNoRange
		val.Done <- true
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package core

import (
	"time"

	"k8s.io/autoscaler/cluster-autoscaler/clusterstate/utils"
	"k8s.io/autoscaler/cluster-autoscaler/metrics"
	"k8s.io/autoscaler/cluster-autoscaler/simulator"
	"k8s.io/autoscaler/cluster-autoscaler/utils/errors"
	kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes"
	kube_client "k8s.io/client-go/kubernetes"
	kube_record "k8s.io/client-go/tools/record"

	"github.com/golang/glog"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
)

// StaticAutoscaler is an autoscaler which has all the core functionality of a CA but without the reconfiguration feature
type StaticAutoscaler struct {
	// AutoscalingContext consists of validated settings and options for this autoscaler
	*AutoscalingContext
	kube_util.ListerRegistry
	lastScaleUpTime          time.Time
	lastScaleDownFailedTrial time.Time
	scaleDown                *ScaleDown
}

// NewStaticAutoscaler creates an instance of Autoscaler filled with provided parameters
func NewStaticAutoscaler(opts AutoscalingOptions, predicateChecker *simulator.PredicateChecker,
	kubeClient kube_client.Interface, kubeEventRecorder kube_record.EventRecorder, listerRegistry kube_util.ListerRegistry) (*StaticAutoscaler, errors.AutoscalerError) {
	logRecorder, err := utils.NewStatusMapRecorder(kubeClient, opts.ConfigNamespace, kubeEventRecorder, opts.WriteStatusConfigMap)
	if err != nil {
		glog.Error("Failed to initialize status configmap, unable to write status events")
		// Get a dummy, so we can at least safely call the methods
		// TODO(maciekpytel): recover from this after successfull status configmap update?
		logRecorder, _ = utils.NewStatusMapRecorder(kubeClient, opts.ConfigNamespace, kubeEventRecorder, false)
	}
	autoscalingContext, errctx := NewAutoscalingContext(opts, predicateChecker, kubeClient, kubeEventRecorder, logRecorder, listerRegistry)
	if errctx != nil {
		return nil, errctx
	}

	scaleDown := NewScaleDown(autoscalingContext)

	return &StaticAutoscaler{
		AutoscalingContext:       autoscalingContext,
		ListerRegistry:           listerRegistry,
		lastScaleUpTime:          time.Now(),
		lastScaleDownFailedTrial: time.Now(),
		scaleDown:                scaleDown,
	}, nil
}

// CleanUp cleans up ToBeDeleted taints added by the previously run and then failed CA
func (a *StaticAutoscaler) CleanUp() {
	// CA can die at any time. Removing taints that might have been left from the previous run.
	if readyNodes, err := a.ReadyNodeLister().List(); err != nil {
		cleanToBeDeleted(readyNodes, a.AutoscalingContext.ClientSet, a.Recorder)
	}
}

// CloudProvider returns the cloud provider associated to this autoscaler
func (a *StaticAutoscaler) CloudProvider() cloudprovider.CloudProvider {
	return a.AutoscalingContext.CloudProvider
}

// RunOnce iterates over node groups and scales them up/down if necessary
func (a *StaticAutoscaler) RunOnce(currentTime time.Time) errors.AutoscalerError {
	readyNodeLister := a.ReadyNodeLister()
	allNodeLister := a.AllNodeLister()
	unschedulablePodLister := a.UnschedulablePodLister()
	scheduledPodLister := a.ScheduledPodLister()
	pdbLister := a.PodDisruptionBudgetLister()
	scaleDown := a.scaleDown
	autoscalingContext := a.AutoscalingContext
	runStart := time.Now()

	readyNodes, err := readyNodeLister.List()
	if err != nil {
		glog.Errorf("Failed to list ready nodes: %v", err)
		return errors.ToAutoscalerError(errors.ApiCallError, err)
	}
	if len(readyNodes) == 0 {
		glog.Warningf("No ready nodes in the cluster")
		scaleDown.CleanUpUnneededNodes()
		return nil
	}

	allNodes, err := allNodeLister.List()
	if err != nil {
		glog.Errorf("Failed to list all nodes: %v", err)
		return errors.ToAutoscalerError(errors.ApiCallError, err)
	}
	if len(allNodes) == 0 {
		glog.Warningf("No nodes in the cluster")
		scaleDown.CleanUpUnneededNodes()
		return nil
	}

	err = a.ClusterStateRegistry.UpdateNodes(allNodes, currentTime)
	if err != nil {
		glog.Errorf("Failed to update node registry: %v", err)
		scaleDown.CleanUpUnneededNodes()
		return errors.ToAutoscalerError(errors.CloudProviderError, err)
	}
	metrics.UpdateClusterState(a.ClusterStateRegistry)

	// Update status information when the loop is done (regardless of reason)
	defer func() {
		if autoscalingContext.WriteStatusConfigMap {
			status := a.ClusterStateRegistry.GetStatus(currentTime)
			utils.WriteStatusConfigMap(autoscalingContext.ClientSet, autoscalingContext.ConfigNamespace,
				status.GetReadableString(), a.AutoscalingContext.LogRecorder)
		}
	}()
	if !a.ClusterStateRegistry.IsClusterHealthy() {
		glog.Warning("Cluster is not ready for autoscaling")
		scaleDown.CleanUpUnneededNodes()
		return nil
	}

	metrics.UpdateDurationFromStart(metrics.UpdateState, runStart)
	metrics.UpdateLastTime(metrics.Autoscaling, time.Now())

	// Check if there are any nodes that failed to register in Kubernetes
	// master.
	unregisteredNodes := a.ClusterStateRegistry.GetUnregisteredNodes()
	if len(unregisteredNodes) > 0 {
		glog.V(1).Infof("%d unregistered nodes present", len(unregisteredNodes))
		removedAny, err := removeOldUnregisteredNodes(unregisteredNodes, autoscalingContext, currentTime)
		// There was a problem with removing unregistered nodes. Retry in the next loop.
		if err != nil {
			if removedAny {
				glog.Warningf("Some unregistered nodes were removed, but got error: %v", err)
			} else {
				glog.Errorf("Failed to remove unregistered nodes: %v", err)

			}
			return errors.ToAutoscalerError(errors.CloudProviderError, err)
		}
		// Some nodes were removed. Let's skip this iteration, the next one should be better.
		if removedAny {
			glog.V(0).Infof("Some unregistered nodes were removed, skipping iteration")
			return nil
		}
	}

	// Check if there has been a constant difference between the number of nodes in k8s and
	// the number of nodes on the cloud provider side.
	// TODO: andrewskim - add protection for ready AWS nodes.
	fixedSomething, err := fixNodeGroupSize(autoscalingContext, currentTime)
	if err != nil {
		glog.Errorf("Failed to fix node group sizes: %v", err)
		return errors.ToAutoscalerError(errors.CloudProviderError, err)
	}
	if fixedSomething {
		glog.V(0).Infof("Some node group target size was fixed, skipping the iteration")
		return nil
	}

	allUnschedulablePods, err := unschedulablePodLister.List()
	if err != nil {
		glog.Errorf("Failed to list unscheduled pods: %v", err)
		return errors.ToAutoscalerError(errors.ApiCallError, err)
	}
	metrics.UpdateUnschedulablePodsCount(len(allUnschedulablePods))

	allScheduled, err := scheduledPodLister.List()
	if err != nil {
		glog.Errorf("Failed to list scheduled pods: %v", err)
		return errors.ToAutoscalerError(errors.ApiCallError, err)
	}

	ConfigurePredicateCheckerForLoop(allUnschedulablePods, allScheduled, a.PredicateChecker)

	// We need to check whether pods marked as unschedulable are actually unschedulable.
	// It's likely we added a new node and the scheduler just haven't managed to put the
	// pod on in yet. In this situation we don't want to trigger another scale-up.
	//
	// It's also important to prevent uncontrollable cluster growth if CA's simulated
	// scheduler differs in opinion with real scheduler. Example of such situation:
	// - CA and Scheduler has slightly different configuration
	// - Scheduler can't schedule a pod and marks it as unschedulable
	// - CA added a node which should help the pod
	// - Scheduler doesn't schedule the pod on the new node
	//   because according to it logic it doesn't fit there
	// - CA see the pod is still unschedulable, so it adds another node to help it
	//
	// With the check enabled the last point won't happen because CA will ignore a pod
	// which is supposed to schedule on an existing node.
	schedulablePodsPresent := false

	glog.V(4).Infof("Filtering out schedulables")
	filterOutSchedulableStart := time.Now()
	unschedulablePodsToHelp := FilterOutSchedulable(allUnschedulablePods, readyNodes, allScheduled,
		a.PredicateChecker)
	metrics.UpdateDurationFromStart(metrics.FilterOutSchedulable, filterOutSchedulableStart)

	if len(unschedulablePodsToHelp) != len(allUnschedulablePods) {
		glog.V(2).Info("Schedulable pods present")
		schedulablePodsPresent = true
	} else {
		glog.V(4).Info("No schedulable pods")
	}

	if len(unschedulablePodsToHelp) == 0 {
		glog.V(1).Info("No unschedulable pods")
	} else if a.MaxNodesTotal > 0 && len(readyNodes) >= a.MaxNodesTotal {
		glog.V(1).Info("Max total nodes in cluster reached")
	} else {
		daemonsets, err := a.ListerRegistry.DaemonSetLister().List()
		if err != nil {
			glog.Errorf("Failed to get daemonset list")
			return errors.ToAutoscalerError(errors.ApiCallError, err)
		}

		scaleUpStart := time.Now()
		metrics.UpdateLastTime(metrics.ScaleUp, scaleUpStart)

		scaledUp, typedErr := ScaleUp(autoscalingContext, unschedulablePodsToHelp, readyNodes, daemonsets)

		metrics.UpdateDurationFromStart(metrics.ScaleUp, scaleUpStart)

		if typedErr != nil {
			glog.Errorf("Failed to scale up: %v", typedErr)
			return typedErr
		} else if scaledUp {
			a.lastScaleUpTime = currentTime
			// No scale down in this iteration.
			return nil
		}
	}

	if a.ScaleDownEnabled {
		pdbs, err := pdbLister.List()
		if err != nil {
			glog.Errorf("Failed to list pod disruption budgets: %v", err)
			return errors.ToAutoscalerError(errors.ApiCallError, err)
		}

		unneededStart := time.Now()
		// In dry run only utilization is updated
		calculateUnneededOnly := a.lastScaleUpTime.Add(a.ScaleDownDelay).After(currentTime) ||
			a.lastScaleDownFailedTrial.Add(a.ScaleDownTrialInterval).After(currentTime) ||
			schedulablePodsPresent ||
			scaleDown.nodeDeleteStatus.IsDeleteInProgress()

		glog.V(4).Infof("Scale down status: unneededOnly=%v lastScaleUpTime=%s "+
			"lastScaleDownFailedTrail=%s schedulablePodsPresent=%v", calculateUnneededOnly,
			a.lastScaleUpTime, a.lastScaleDownFailedTrial, schedulablePodsPresent)

		glog.V(4).Infof("Calculating unneeded nodes")

		scaleDown.CleanUp(currentTime)
		potentiallyUnneeded := getPotentiallyUnneededNodes(autoscalingContext, allNodes)

		typedErr := scaleDown.UpdateUnneededNodes(allNodes, potentiallyUnneeded, allScheduled, currentTime, pdbs)
		if typedErr != nil {
			glog.Errorf("Failed to scale down: %v", typedErr)
			return typedErr
		}

		metrics.UpdateDurationFromStart(metrics.FindUnneeded, unneededStart)

		for key, val := range scaleDown.unneededNodes {
			if glog.V(4) {
				glog.V(4).Infof("%s is unneeded since %s duration %s", key, val.String(), time.Now().Sub(val).String())
			}
		}

		if !calculateUnneededOnly {
			glog.V(4).Infof("Starting scale down")

			// We want to delete unneeded Node Groups only if there was no recent scale up,
			// and there is no current delete in progress and there was no recent errors.
			if a.AutoscalingContext.NodeAutoprovisioningEnabled {
				err := cleanUpNodeAutoprovisionedGroups(a.AutoscalingContext.CloudProvider)
				if err != nil {
					glog.Warningf("Failed to clean up unneded node groups: %v", err)
				}
			}

			scaleDownStart := time.Now()
			metrics.UpdateLastTime(metrics.ScaleDown, scaleDownStart)
			result, typedErr := scaleDown.TryToScaleDown(allNodes, allScheduled, pdbs, currentTime)
			metrics.UpdateDurationFromStart(metrics.ScaleDown, scaleDownStart)

			// TODO: revisit result handling
			if typedErr != nil {
				glog.Errorf("Failed to scale down: %v", err)
				return typedErr
			}
			if result == ScaleDownError {
				a.lastScaleDownFailedTrial = currentTime
			}
		}
	}
	return nil
}

// ExitCleanUp removes status configmap.
func (a *StaticAutoscaler) ExitCleanUp() {
	if !a.AutoscalingContext.WriteStatusConfigMap {
		return
	}
	utils.DeleteStatusConfigMap(a.AutoscalingContext.ClientSet, a.AutoscalingContext.ConfigNamespace)
}
<commit_msg>Move calculateUnneededOnly check after unneeded calculations, add log message to main loop start<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package core

import (
	"time"

	"k8s.io/autoscaler/cluster-autoscaler/clusterstate/utils"
	"k8s.io/autoscaler/cluster-autoscaler/metrics"
	"k8s.io/autoscaler/cluster-autoscaler/simulator"
	"k8s.io/autoscaler/cluster-autoscaler/utils/errors"
	kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes"
	kube_client "k8s.io/client-go/kubernetes"
	kube_record "k8s.io/client-go/tools/record"

	"github.com/golang/glog"
	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
)

// StaticAutoscaler is an autoscaler which has all the core functionality of a CA but without the reconfiguration feature
type StaticAutoscaler struct {
	// AutoscalingContext consists of validated settings and options for this autoscaler
	*AutoscalingContext
	kube_util.ListerRegistry
	lastScaleUpTime          time.Time
	lastScaleDownFailedTrial time.Time
	scaleDown                *ScaleDown
}

// NewStaticAutoscaler creates an instance of Autoscaler filled with provided parameters
func NewStaticAutoscaler(opts AutoscalingOptions, predicateChecker *simulator.PredicateChecker,
	kubeClient kube_client.Interface, kubeEventRecorder kube_record.EventRecorder, listerRegistry kube_util.ListerRegistry) (*StaticAutoscaler, errors.AutoscalerError) {
	logRecorder, err := utils.NewStatusMapRecorder(kubeClient, opts.ConfigNamespace, kubeEventRecorder, opts.WriteStatusConfigMap)
	if err != nil {
		glog.Error("Failed to initialize status configmap, unable to write status events")
		// Get a dummy, so we can at least safely call the methods
		// TODO(maciekpytel): recover from this after successfull status configmap update?
		logRecorder, _ = utils.NewStatusMapRecorder(kubeClient, opts.ConfigNamespace, kubeEventRecorder, false)
	}
	autoscalingContext, errctx := NewAutoscalingContext(opts, predicateChecker, kubeClient, kubeEventRecorder, logRecorder, listerRegistry)
	if errctx != nil {
		return nil, errctx
	}

	scaleDown := NewScaleDown(autoscalingContext)

	return &StaticAutoscaler{
		AutoscalingContext:       autoscalingContext,
		ListerRegistry:           listerRegistry,
		lastScaleUpTime:          time.Now(),
		lastScaleDownFailedTrial: time.Now(),
		scaleDown:                scaleDown,
	}, nil
}

// CleanUp cleans up ToBeDeleted taints added by the previously run and then failed CA
func (a *StaticAutoscaler) CleanUp() {
	// CA can die at any time. Removing taints that might have been left from the previous run.
	if readyNodes, err := a.ReadyNodeLister().List(); err != nil {
		cleanToBeDeleted(readyNodes, a.AutoscalingContext.ClientSet, a.Recorder)
	}
}

// CloudProvider returns the cloud provider associated to this autoscaler
func (a *StaticAutoscaler) CloudProvider() cloudprovider.CloudProvider {
	return a.AutoscalingContext.CloudProvider
}

// RunOnce iterates over node groups and scales them up/down if necessary
func (a *StaticAutoscaler) RunOnce(currentTime time.Time) errors.AutoscalerError {
	readyNodeLister := a.ReadyNodeLister()
	allNodeLister := a.AllNodeLister()
	unschedulablePodLister := a.UnschedulablePodLister()
	scheduledPodLister := a.ScheduledPodLister()
	pdbLister := a.PodDisruptionBudgetLister()
	scaleDown := a.scaleDown
	autoscalingContext := a.AutoscalingContext
	runStart := time.Now()

	glog.V(4).Info("Starting main loop")

	readyNodes, err := readyNodeLister.List()
	if err != nil {
		glog.Errorf("Failed to list ready nodes: %v", err)
		return errors.ToAutoscalerError(errors.ApiCallError, err)
	}
	if len(readyNodes) == 0 {
		glog.Warningf("No ready nodes in the cluster")
		scaleDown.CleanUpUnneededNodes()
		return nil
	}

	allNodes, err := allNodeLister.List()
	if err != nil {
		glog.Errorf("Failed to list all nodes: %v", err)
		return errors.ToAutoscalerError(errors.ApiCallError, err)
	}
	if len(allNodes) == 0 {
		glog.Warningf("No nodes in the cluster")
		scaleDown.CleanUpUnneededNodes()
		return nil
	}

	err = a.ClusterStateRegistry.UpdateNodes(allNodes, currentTime)
	if err != nil {
		glog.Errorf("Failed to update node registry: %v", err)
		scaleDown.CleanUpUnneededNodes()
		return errors.ToAutoscalerError(errors.CloudProviderError, err)
	}
	metrics.UpdateClusterState(a.ClusterStateRegistry)

	// Update status information when the loop is done (regardless of reason)
	defer func() {
		if autoscalingContext.WriteStatusConfigMap {
			status := a.ClusterStateRegistry.GetStatus(currentTime)
			utils.WriteStatusConfigMap(autoscalingContext.ClientSet, autoscalingContext.ConfigNamespace,
				status.GetReadableString(), a.AutoscalingContext.LogRecorder)
		}
	}()
	if !a.ClusterStateRegistry.IsClusterHealthy() {
		glog.Warning("Cluster is not ready for autoscaling")
		scaleDown.CleanUpUnneededNodes()
		return nil
	}

	metrics.UpdateDurationFromStart(metrics.UpdateState, runStart)
	metrics.UpdateLastTime(metrics.Autoscaling, time.Now())

	// Check if there are any nodes that failed to register in Kubernetes
	// master.
	unregisteredNodes := a.ClusterStateRegistry.GetUnregisteredNodes()
	if len(unregisteredNodes) > 0 {
		glog.V(1).Infof("%d unregistered nodes present", len(unregisteredNodes))
		removedAny, err := removeOldUnregisteredNodes(unregisteredNodes, autoscalingContext, currentTime)
		// There was a problem with removing unregistered nodes. Retry in the next loop.
		if err != nil {
			if removedAny {
				glog.Warningf("Some unregistered nodes were removed, but got error: %v", err)
			} else {
				glog.Errorf("Failed to remove unregistered nodes: %v", err)

			}
			return errors.ToAutoscalerError(errors.CloudProviderError, err)
		}
		// Some nodes were removed. Let's skip this iteration, the next one should be better.
		if removedAny {
			glog.V(0).Infof("Some unregistered nodes were removed, skipping iteration")
			return nil
		}
	}

	// Check if there has been a constant difference between the number of nodes in k8s and
	// the number of nodes on the cloud provider side.
	// TODO: andrewskim - add protection for ready AWS nodes.
	fixedSomething, err := fixNodeGroupSize(autoscalingContext, currentTime)
	if err != nil {
		glog.Errorf("Failed to fix node group sizes: %v", err)
		return errors.ToAutoscalerError(errors.CloudProviderError, err)
	}
	if fixedSomething {
		glog.V(0).Infof("Some node group target size was fixed, skipping the iteration")
		return nil
	}

	allUnschedulablePods, err := unschedulablePodLister.List()
	if err != nil {
		glog.Errorf("Failed to list unscheduled pods: %v", err)
		return errors.ToAutoscalerError(errors.ApiCallError, err)
	}
	metrics.UpdateUnschedulablePodsCount(len(allUnschedulablePods))

	allScheduled, err := scheduledPodLister.List()
	if err != nil {
		glog.Errorf("Failed to list scheduled pods: %v", err)
		return errors.ToAutoscalerError(errors.ApiCallError, err)
	}

	ConfigurePredicateCheckerForLoop(allUnschedulablePods, allScheduled, a.PredicateChecker)

	// We need to check whether pods marked as unschedulable are actually unschedulable.
	// It's likely we added a new node and the scheduler just haven't managed to put the
	// pod on in yet. In this situation we don't want to trigger another scale-up.
	//
	// It's also important to prevent uncontrollable cluster growth if CA's simulated
	// scheduler differs in opinion with real scheduler. Example of such situation:
	// - CA and Scheduler has slightly different configuration
	// - Scheduler can't schedule a pod and marks it as unschedulable
	// - CA added a node which should help the pod
	// - Scheduler doesn't schedule the pod on the new node
	//   because according to it logic it doesn't fit there
	// - CA see the pod is still unschedulable, so it adds another node to help it
	//
	// With the check enabled the last point won't happen because CA will ignore a pod
	// which is supposed to schedule on an existing node.
	schedulablePodsPresent := false

	glog.V(4).Infof("Filtering out schedulables")
	filterOutSchedulableStart := time.Now()
	unschedulablePodsToHelp := FilterOutSchedulable(allUnschedulablePods, readyNodes, allScheduled,
		a.PredicateChecker)
	metrics.UpdateDurationFromStart(metrics.FilterOutSchedulable, filterOutSchedulableStart)

	if len(unschedulablePodsToHelp) != len(allUnschedulablePods) {
		glog.V(2).Info("Schedulable pods present")
		schedulablePodsPresent = true
	} else {
		glog.V(4).Info("No schedulable pods")
	}

	if len(unschedulablePodsToHelp) == 0 {
		glog.V(1).Info("No unschedulable pods")
	} else if a.MaxNodesTotal > 0 && len(readyNodes) >= a.MaxNodesTotal {
		glog.V(1).Info("Max total nodes in cluster reached")
	} else {
		daemonsets, err := a.ListerRegistry.DaemonSetLister().List()
		if err != nil {
			glog.Errorf("Failed to get daemonset list")
			return errors.ToAutoscalerError(errors.ApiCallError, err)
		}

		scaleUpStart := time.Now()
		metrics.UpdateLastTime(metrics.ScaleUp, scaleUpStart)

		scaledUp, typedErr := ScaleUp(autoscalingContext, unschedulablePodsToHelp, readyNodes, daemonsets)

		metrics.UpdateDurationFromStart(metrics.ScaleUp, scaleUpStart)

		if typedErr != nil {
			glog.Errorf("Failed to scale up: %v", typedErr)
			return typedErr
		} else if scaledUp {
			a.lastScaleUpTime = currentTime
			// No scale down in this iteration.
			return nil
		}
	}

	if a.ScaleDownEnabled {
		pdbs, err := pdbLister.List()
		if err != nil {
			glog.Errorf("Failed to list pod disruption budgets: %v", err)
			return errors.ToAutoscalerError(errors.ApiCallError, err)
		}

		unneededStart := time.Now()

		glog.V(4).Infof("Calculating unneeded nodes")

		scaleDown.CleanUp(currentTime)
		potentiallyUnneeded := getPotentiallyUnneededNodes(autoscalingContext, allNodes)

		typedErr := scaleDown.UpdateUnneededNodes(allNodes, potentiallyUnneeded, allScheduled, currentTime, pdbs)
		if typedErr != nil {
			glog.Errorf("Failed to scale down: %v", typedErr)
			return typedErr
		}

		metrics.UpdateDurationFromStart(metrics.FindUnneeded, unneededStart)

		for key, val := range scaleDown.unneededNodes {
			if glog.V(4) {
				glog.V(4).Infof("%s is unneeded since %s duration %s", key, val.String(), time.Now().Sub(val).String())
			}
		}

		// In dry run only utilization is updated
		calculateUnneededOnly := a.lastScaleUpTime.Add(a.ScaleDownDelay).After(currentTime) ||
			a.lastScaleDownFailedTrial.Add(a.ScaleDownTrialInterval).After(currentTime) ||
			schedulablePodsPresent ||
			scaleDown.nodeDeleteStatus.IsDeleteInProgress()

		glog.V(4).Infof("Scale down status: unneededOnly=%v lastScaleUpTime=%s "+
			"lastScaleDownFailedTrail=%s schedulablePodsPresent=%v", calculateUnneededOnly,
			a.lastScaleUpTime, a.lastScaleDownFailedTrial, schedulablePodsPresent)

		if !calculateUnneededOnly {
			glog.V(4).Infof("Starting scale down")

			// We want to delete unneeded Node Groups only if there was no recent scale up,
			// and there is no current delete in progress and there was no recent errors.
			if a.AutoscalingContext.NodeAutoprovisioningEnabled {
				err := cleanUpNodeAutoprovisionedGroups(a.AutoscalingContext.CloudProvider)
				if err != nil {
					glog.Warningf("Failed to clean up unneded node groups: %v", err)
				}
			}

			scaleDownStart := time.Now()
			metrics.UpdateLastTime(metrics.ScaleDown, scaleDownStart)
			result, typedErr := scaleDown.TryToScaleDown(allNodes, allScheduled, pdbs, currentTime)
			metrics.UpdateDurationFromStart(metrics.ScaleDown, scaleDownStart)

			// TODO: revisit result handling
			if typedErr != nil {
				glog.Errorf("Failed to scale down: %v", err)
				return typedErr
			}
			if result == ScaleDownError {
				a.lastScaleDownFailedTrial = currentTime
			}
		}
	}
	return nil
}

// ExitCleanUp removes status configmap.
func (a *StaticAutoscaler) ExitCleanUp() {
	if !a.AutoscalingContext.WriteStatusConfigMap {
		return
	}
	utils.DeleteStatusConfigMap(a.AutoscalingContext.ClientSet, a.AutoscalingContext.ConfigNamespace)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"unicode/utf8"
)

const (
	documentDelimiter         = "---"
	minusIndentationCharacter = "-"
	spaceIndentationCharacter = " "
	indentationCharacters     = minusIndentationCharacter + spaceIndentationCharacter
	commentCharacters         = "#"
	defaultIndentationLength  = 2 //used only when heuristic fails to provide indentation lengths used in input file
)

var (
	deploymentKind = regexp.MustCompile(`kind: *Deployment`)
	podKind        = regexp.MustCompile(`kind: *Pod`)
)

// inject injects yaml file content with ldpreload labels
func inject(content string, params injectParams) (string, error) {
	eol, err := detectEOLString(content)
	if err != nil {
		return "", err
	}

	var converted bytes.Buffer
	for _, document := range strings.Split(content, documentDelimiter) {
		if converted.Len() != 0 {
			converted.WriteString(documentDelimiter)
		}
		if isPod(document) || isDeployment(document) {
			document = insertLDPreloadTrue(document, eol)
			document = insertAppScope(document, eol, params)
			if params.useDebugLabel {
				document = insertDebug(document, eol)
			}
			converted.WriteString(document)
		} else {
			converted.WriteString(document)
		}
	}
	return converted.String(), nil
}

// insertion is complete information needed for injection of one string into yaml file content string
type insertion struct {
	insertionPoint int
	text           string
}

func insertLDPreloadTrue(document string, eol string) string {
	return insertLines(
		document,
		[]string{"spec:", "template:", "metadata:", "labels:"},
		[]string{
			"# ldpreload-related labels",
			"ldpreload: \"true\"",
		},
		eol)
}

func insertAppScope(document string, eol string, params injectParams) string {
	if params.proxyName == "" { // no proxy -> scope is global for all containers
		return insertLines(
			document,
			[]string{"spec:", "template:", "spec:", "containers:", "-", "env:"},
			[]string{
				"# ldpreload-related env vars",
				"- name: VCL_APP_SCOPE_GLOBAL",
				"  value: \"\"",
			},
			eol)
	}

	// proxy used
	return insertLinesConditioned(
		document,
		[]string{"spec:", "template:", "spec:", "containers:", "-", "env:"},
		isProxyContainer(params.proxyName),
		[]string{ // proxy container settings
			"# ldpreload-related env vars",
			"- name: VCL_APP_SCOPE_GLOBAL",
			"  value: \"\"",
			"- name: VCL_APP_SCOPE_LOCAL",
			"  value: \"\"",
			"- name: VCL_APP_PROXY_TRANSPORT_TCP",
			"  value: \"\"",
		},
		[]string{ // non-proxy container settings
			"# ldpreload-related env vars",
			"- name: VCL_APP_SCOPE_LOCAL",
			"  value: \"\"",
		},
		eol,
	)
}

// isProxyContainer checks if we are inside proxy image or not. This impl is tightly relying on path in insertAppScope() method.
func isProxyContainer(proxyName string) conditionFunc {
	return func(info traversingInfo) bool {
		if len(info.unresolvedPath) > 1 { //TODO handle no container situation
			return false // should not happen, because it would mean that there are no containers at all
		}

		imageBlockStr := info.blockStr(len(info.blocks) - (2 - len(info.unresolvedPath)))
		return regexp.MustCompile("name: *"+proxyName+" *"+info.eol).
			FindStringIndex(imageBlockStr) != nil
	}
}

func insertDebug(document string, eol string) string {
	return insertLines(
		document,
		[]string{"spec:", "template:", "spec:", "containers:", "-", "env:"},
		[]string{
			"# enable verbose VCL debugs, do not use for production",
			"- name: VCL_DEBUG",
			"  value: \"3\"",
		},
		eol)
}

func insertLines(document string, path []string, insertLines []string, eol string) string {
	return insertLinesConditioned(document, path, func(traversingInfo) bool { return true }, insertLines, []string{}, eol)
}

type conditionFunc func(traversingInfo) bool

func insertLinesConditioned(document string, path []string, condition conditionFunc, insertLines []string, elseInsertLines []string, eol string) string {
	var insertions = &[]insertion{}
	visitInsertionPlaces(newTraversingInfo(document, path,
		func(i traversingInfo) {
			lines := insertLines
			if !condition(i) {
				lines = elseInsertLines
			}
			insertStr := createIndentationedInsertionString(i, eol, lines)
			//TODO check existence before inserting
			insertPoint := strings.Index(i.curBlockStr(), eol) + len(eol) + i.curBlockStart()
			insertions = prepend(insertion{insertPoint, insertStr}, insertions) // using this order doesn't invalidate indexes of insertions by applying them sequentially
			return
		}, eol))

	//make real insert
	for _, insert := range *insertions {
		document = document[:insert.insertionPoint] + insert.text + document[insert.insertionPoint:]
	}
	return document
}
func createIndentationedInsertionString(i traversingInfo, eol string, insertLines []string) string {
	//computing indentation delta additions to define inner block indentation
	indentationDelta := defaultIndentationLength // just guess in case we don't have enough information to compute it
	if len(i.resolvedPath) > 1 {
		indentationDelta = round(float64(i.parentBlockIndentation) / float64(len(i.resolvedPath)-1))
	}

	//compute indentation of inner block
	indentation := 0 //default is for case when len(i.resolvedPath) == 0
	if len(i.resolvedPath) > 0 {
		// checking indentation of siblings
		match := regexp.
			MustCompile(i.eol + "([" + spaceIndentationCharacter + "]*?)[^" + spaceIndentationCharacter + commentCharacters + "]{1}").
			FindStringSubmatch(i.curBlockStr())
		if match != nil {
			indentation = len(match[1])
		} else { //no siblings -> using heuristic
			indentation = i.parentBlockIndentation + indentationDelta
		}
	}

	//creating missing path if necessary
	var buffer bytes.Buffer
	if len(i.unresolvedPath) > 0 {
		for _, pathPart := range i.unresolvedPath {
			buffer.WriteString(strings.Repeat(" ", indentation) + pathPart + eol)
			indentation = indentation + indentationDelta
		}
	}

	//creating insertion lines with proper indentation
	for _, line := range insertLines {
		buffer.WriteString(strings.Repeat(" ", indentation) + line + eol)
	}

	return buffer.String()
}

func prepend(item insertion, slice *[]insertion) *[]insertion {
	newSlice := append([]insertion{item}, *slice...)
	return &newSlice
}

type traversingInfo struct {
	// static info that doesn't change by traversing
	document string
	visitor  func(traversingInfo) //passing copy only (slices can still refer back to original array)
	eol      string

	// dynamic info changed by traversing
	unresolvedPath         []string
	resolvedPath           []string
	blocks                 []block
	parentBlockIndentation int
}

type block struct {
	start int
	end   int
}

func newTraversingInfo(document string, path []string, visitor func(traversingInfo), eol string) traversingInfo {
	return traversingInfo{
		document: document,
		visitor:  visitor,
		eol:      eol,

		unresolvedPath:         path,
		resolvedPath:           []string{},
		blocks:                 []block{{0, len(document)}},
		parentBlockIndentation: 0,
	}
}

func (t *traversingInfo) newDescending(blockStart int, blockEnd int, parentBlockIndentation int) traversingInfo {
	return traversingInfo{
		document: t.document,
		visitor:  t.visitor,
		eol:      t.eol,

		unresolvedPath:         t.unresolvedPath[1:],
		resolvedPath:           append(t.resolvedPath, t.unresolvedPath[0]),
		blocks:                 append(t.blocks, block{blockStart, blockEnd}),
		parentBlockIndentation: parentBlockIndentation,
	}
}

func (t *traversingInfo) curBlock() block {
	return t.blocks[len(t.blocks)-1]
}

func (t *traversingInfo) curBlockStart() int {
	return t.curBlock().start
}

func (t *traversingInfo) curBlockEnd() int {
	return t.curBlock().end
}

func (t *traversingInfo) curBlockStr() string {
	return t.document[t.curBlockStart():t.curBlockEnd()]
}

func (t *traversingInfo) blockStr(blockIndex int) string {
	return t.document[t.blocks[blockIndex].start:t.blocks[blockIndex].end]
}

func visitInsertionPlaces(i traversingInfo) {
	if len(i.unresolvedPath) == 0 {
		i.visitor(i)
		return
	}

	if i.unresolvedPath[0] == "-" { // compact nested mapping
		blockIndentation := computeMappingBlockIndentation(i.curBlockStr(), i.eol)
		mappingItemPrefixes := regexp.
			MustCompile(i.eol+"["+spaceIndentationCharacter+"]{"+strconv.Itoa(blockIndentation)+"}"+minusIndentationCharacter).
			FindAllStringIndex(i.curBlockStr(), -1)
		for _, prefixIndexes := range mappingItemPrefixes {
			itemBlockStart, itemBlockEnd := computeItemBlockPosition(i.curBlockStr(), prefixIndexes, blockIndentation, i.eol)

			// recursive call would not handle map item block correctly => handling 1 recursive call here (recursive calls
			// can continue when in mapping items are normal blocks again)
			// Expecting that unresolved paths can't end with "-"
			childBlockIndentation := computeItemBlockIndentation(i.curBlockStr(), i.eol)
			handleBasicBlock(i.newDescending(i.curBlockStart()+itemBlockStart, i.curBlockStart()+itemBlockEnd, blockIndentation), childBlockIndentation)
		}
	} else { // basic blocks
		blockIndentation := computeNormalBlockIndentation(i.curBlockStr(), i.eol)
		handleBasicBlock(i, blockIndentation)
	}
}

func handleBasicBlock(i traversingInfo, blockIndentation int) {
	matchedBlockPrefixes := regexp.
		MustCompile(i.eol+"["+indentationCharacters+"]{"+strconv.Itoa(blockIndentation)+"}"+i.unresolvedPath[0]).
		FindAllStringIndex(i.curBlockStr(), -1)
	if len(matchedBlockPrefixes) == 0 { //next block doesn't exist
		i.visitor(i)
		return
	}
	for _, prefixIndexes := range matchedBlockPrefixes {
		childBlockStart, childBlockEnd := computeChildBlockPosition(i.curBlockStr(), prefixIndexes, blockIndentation, i.eol)
		visitInsertionPlaces(i.newDescending(i.curBlockStart()+childBlockStart, i.curBlockStart()+childBlockEnd, blockIndentation))
	}
}

func computeNormalBlockIndentation(curBlock string, eol string) int {
	return computeBlockIndentation(curBlock, eol, eol+"["+spaceIndentationCharacter+"]*[^"+indentationCharacters+commentCharacters+"]{1}") //TODO convert indentationCharacters to regexp? for "- " is it the same
}

func computeMappingBlockIndentation(curBlock string, eol string) int {
	return computeBlockIndentation(curBlock, eol, eol+"["+spaceIndentationCharacter+"]*"+minusIndentationCharacter) //TODO convert indentationCharacters to regexp? for "- " is it the same
}

func computeItemBlockIndentation(curBlock string, eol string) int {
	return computeBlockIndentation(curBlock, eol, eol+"["+indentationCharacters+"]*[^"+indentationCharacters+commentCharacters+"]{1}") //TODO convert indentationCharacters to regexp? for "- " is it the same
}

func computeBlockIndentation(curBlock string, eol string, indentationRegExp string) int {
	indentation := regexp.MustCompile(indentationRegExp).FindString(curBlock)
	if indentation == "" { //block has no child blocks
		return -1
	}
	_, lastRuneSize := utf8.DecodeLastRuneInString(indentation)
	return len(indentation) - len(eol) - lastRuneSize //TODO convert byte length to rune length? for " " and "-" it is the same length
}

func computeChildBlockPosition(curBlock string, childBlockPrefixIndexes []int, blockIndentation int, eol string) (int, int) {
	return computeInnerBlockPosition(curBlock, childBlockPrefixIndexes, blockIndentation, eol, "[^"+indentationCharacters+commentCharacters+"]{1}")
}

func computeItemBlockPosition(curBlock string, itemBlockPrefixIndexes []int, blockIndentation int, eol string) (int, int) {
	return computeInnerBlockPosition(curBlock, itemBlockPrefixIndexes, blockIndentation, eol, minusIndentationCharacter)
}

func computeInnerBlockPosition(curBlock string, innerBlockPrefixIndexes []int, blockIndentation int, eol string, lastCharacterRegExp string) (int, int) {
	start := innerBlockPrefixIndexes[0] + len(eol)                                                                           //without eol from previous block
	nextSibling := eol + "[" + spaceIndentationCharacter + "]{" + strconv.Itoa(blockIndentation) + "}" + lastCharacterRegExp //TODO convert indentationCharacters to regexp? for "- " is it the same
	nextSiblingIdx := regexp.MustCompile(nextSibling).FindStringIndex(curBlock[innerBlockPrefixIndexes[0]+len(eol):])        // shifted search by len(eol) to not match start of block
	if nextSiblingIdx != nil {
		return start, innerBlockPrefixIndexes[0] + nextSiblingIdx[0] + len(eol) /*shifted sibling search*/ + len(eol) /*include block ending eol (matched by sibling search) */
	}
	return start, len(curBlock)
}

func detectEOLString(content string) (string, error) {
	for _, eol := range []string{"\r\n", "\n", "\r"} {
		if strings.Contains(content, eol) {
			return eol, nil
		}
	}
	return "", fmt.Errorf("can't detect end of line characters")
}

func isDeployment(document string) bool {
	return len(deploymentKind.FindString(document)) != 0
}

func isPod(document string) bool {
	return len(podKind.FindString(document)) != 0
}

func round(f float64) int {
	if f < -0.5 {
		return int(f - 0.5)
	}
	if f > 0.5 {
		return int(f + 0.5)
	}
	return 0
}
<commit_msg>applied new knowledge about ldpreload labels inserting<commit_after>package main

import (
	"bytes"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"unicode/utf8"
)

const (
	documentDelimiter         = "---"
	minusIndentationCharacter = "-"
	spaceIndentationCharacter = " "
	indentationCharacters     = minusIndentationCharacter + spaceIndentationCharacter
	commentCharacters         = "#"
	defaultIndentationLength  = 2 //used only when heuristic fails to provide indentation lengths used in input file
)

var (
	deploymentKind = regexp.MustCompile(`kind: *Deployment`)
	podKind        = regexp.MustCompile(`kind: *Pod`)
)

// inject injects yaml file content with ldpreload labels
func inject(content string, params injectParams) (string, error) {
	eol, err := detectEOLString(content)
	if err != nil {
		return "", err
	}

	var converted bytes.Buffer
	for _, document := range strings.Split(content, documentDelimiter) {
		if converted.Len() != 0 {
			converted.WriteString(documentDelimiter)
		}
		if isPod(document) || isDeployment(document) {
			document = insertLDPreloadTrue(document, eol)
			document = insertAppScope(document, eol, params)
			if params.useDebugLabel {
				document = insertDebug(document, eol)
			}
			converted.WriteString(document)
		} else {
			converted.WriteString(document)
		}
	}
	return converted.String(), nil
}

// insertion is complete information needed for injection of one string into yaml file content string
type insertion struct {
	insertionPoint int
	text           string
}

func insertLDPreloadTrue(document string, eol string) string {
	return insertLines(
		document,
		[]string{"spec:", "template:", "metadata:", "labels:"},
		[]string{
			"# ldpreload-related labels",
			"ldpreload: \"true\"",
		},
		eol)
}

func insertAppScope(document string, eol string, params injectParams) string {
	if params.proxyName == "" { // no proxy -> scope is global for all containers
		return insertLines(
			document,
			[]string{"spec:", "template:", "spec:", "containers:", "-", "env:"},
			[]string{
				"# ldpreload-related env vars",
				"- name: VCL_APP_SCOPE_GLOBAL",
				"  value: \"\"",
				"- name: VCL_APP_SCOPE_LOCAL",
				"  value: \"\"",
			},
			eol)
	}

	// proxy used
	return insertLinesConditioned(
		document,
		[]string{"spec:", "template:", "spec:", "containers:", "-", "env:"},
		isProxyContainer(params.proxyName),
		[]string{ // proxy container settings
			"# ldpreload-related env vars",
			"- name: VCL_APP_SCOPE_GLOBAL",
			"  value: \"\"",
			"- name: VCL_APP_SCOPE_LOCAL",
			"  value: \"\"",
			"- name: VCL_APP_PROXY_TRANSPORT_TCP",
			"  value: \"\"",
		},
		[]string{ // non-proxy container settings
			"# ldpreload-related env vars",
			"- name: VCL_APP_SCOPE_LOCAL",
			"  value: \"\"",
		},
		eol,
	)
}

// isProxyContainer checks if we are inside proxy image or not. This impl is tightly relying on path in insertAppScope() method.
func isProxyContainer(proxyName string) conditionFunc {
	return func(info traversingInfo) bool {
		if len(info.unresolvedPath) > 1 { //TODO handle no container situation
			return false // should not happen, because it would mean that there are no containers at all
		}

		imageBlockStr := info.blockStr(len(info.blocks) - (2 - len(info.unresolvedPath)))
		return regexp.MustCompile("name: *"+proxyName+" *"+info.eol).
			FindStringIndex(imageBlockStr) != nil
	}
}

func insertDebug(document string, eol string) string {
	return insertLines(
		document,
		[]string{"spec:", "template:", "spec:", "containers:", "-", "env:"},
		[]string{
			"# enable verbose VCL debugs, do not use for production",
			"- name: VCL_DEBUG",
			"  value: \"3\"",
		},
		eol)
}

func insertLines(document string, path []string, insertLines []string, eol string) string {
	return insertLinesConditioned(document, path, func(traversingInfo) bool { return true }, insertLines, []string{}, eol)
}

type conditionFunc func(traversingInfo) bool

func insertLinesConditioned(document string, path []string, condition conditionFunc, insertLines []string, elseInsertLines []string, eol string) string {
	var insertions = &[]insertion{}
	visitInsertionPlaces(newTraversingInfo(document, path,
		func(i traversingInfo) {
			lines := insertLines
			if !condition(i) {
				lines = elseInsertLines
			}
			insertStr := createIndentationedInsertionString(i, eol, lines)
			//TODO check existence before inserting
			insertPoint := strings.Index(i.curBlockStr(), eol) + len(eol) + i.curBlockStart()
			insertions = prepend(insertion{insertPoint, insertStr}, insertions) // using this order doesn't invalidate indexes of insertions by applying them sequentially
			return
		}, eol))

	//make real insert
	for _, insert := range *insertions {
		document = document[:insert.insertionPoint] + insert.text + document[insert.insertionPoint:]
	}
	return document
}
func createIndentationedInsertionString(i traversingInfo, eol string, insertLines []string) string {
	//computing indentation delta additions to define inner block indentation
	indentationDelta := defaultIndentationLength // just guess in case we don't have enough information to compute it
	if len(i.resolvedPath) > 1 {
		indentationDelta = round(float64(i.parentBlockIndentation) / float64(len(i.resolvedPath)-1))
	}

	//compute indentation of inner block
	indentation := 0 //default is for case when len(i.resolvedPath) == 0
	if len(i.resolvedPath) > 0 {
		// checking indentation of siblings
		match := regexp.
			MustCompile(i.eol + "([" + spaceIndentationCharacter + "]*?)[^" + spaceIndentationCharacter + commentCharacters + "]{1}").
			FindStringSubmatch(i.curBlockStr())
		if match != nil {
			indentation = len(match[1])
		} else { //no siblings -> using heuristic
			indentation = i.parentBlockIndentation + indentationDelta
		}
	}

	//creating missing path if necessary
	var buffer bytes.Buffer
	if len(i.unresolvedPath) > 0 {
		for _, pathPart := range i.unresolvedPath {
			buffer.WriteString(strings.Repeat(" ", indentation) + pathPart + eol)
			indentation = indentation + indentationDelta
		}
	}

	//creating insertion lines with proper indentation
	for _, line := range insertLines {
		buffer.WriteString(strings.Repeat(" ", indentation) + line + eol)
	}

	return buffer.String()
}

func prepend(item insertion, slice *[]insertion) *[]insertion {
	newSlice := append([]insertion{item}, *slice...)
	return &newSlice
}

type traversingInfo struct {
	// static info that doesn't change by traversing
	document string
	visitor  func(traversingInfo) //passing copy only (slices can still refer back to original array)
	eol      string

	// dynamic info changed by traversing
	unresolvedPath         []string
	resolvedPath           []string
	blocks                 []block
	parentBlockIndentation int
}

type block struct {
	start int
	end   int
}

func newTraversingInfo(document string, path []string, visitor func(traversingInfo), eol string) traversingInfo {
	return traversingInfo{
		document: document,
		visitor:  visitor,
		eol:      eol,

		unresolvedPath:         path,
		resolvedPath:           []string{},
		blocks:                 []block{{0, len(document)}},
		parentBlockIndentation: 0,
	}
}

func (t *traversingInfo) newDescending(blockStart int, blockEnd int, parentBlockIndentation int) traversingInfo {
	return traversingInfo{
		document: t.document,
		visitor:  t.visitor,
		eol:      t.eol,

		unresolvedPath:         t.unresolvedPath[1:],
		resolvedPath:           append(t.resolvedPath, t.unresolvedPath[0]),
		blocks:                 append(t.blocks, block{blockStart, blockEnd}),
		parentBlockIndentation: parentBlockIndentation,
	}
}

func (t *traversingInfo) curBlock() block {
	return t.blocks[len(t.blocks)-1]
}

func (t *traversingInfo) curBlockStart() int {
	return t.curBlock().start
}

func (t *traversingInfo) curBlockEnd() int {
	return t.curBlock().end
}

func (t *traversingInfo) curBlockStr() string {
	return t.document[t.curBlockStart():t.curBlockEnd()]
}

func (t *traversingInfo) blockStr(blockIndex int) string {
	return t.document[t.blocks[blockIndex].start:t.blocks[blockIndex].end]
}

func visitInsertionPlaces(i traversingInfo) {
	if len(i.unresolvedPath) == 0 {
		i.visitor(i)
		return
	}

	if i.unresolvedPath[0] == "-" { // compact nested mapping
		blockIndentation := computeMappingBlockIndentation(i.curBlockStr(), i.eol)
		mappingItemPrefixes := regexp.
			MustCompile(i.eol+"["+spaceIndentationCharacter+"]{"+strconv.Itoa(blockIndentation)+"}"+minusIndentationCharacter).
			FindAllStringIndex(i.curBlockStr(), -1)
		for _, prefixIndexes := range mappingItemPrefixes {
			itemBlockStart, itemBlockEnd := computeItemBlockPosition(i.curBlockStr(), prefixIndexes, blockIndentation, i.eol)

			// recursive call would not handle map item block correctly => handling 1 recursive call here (recursive calls
			// can continue when in mapping items are normal blocks again)
			// Expecting that unresolved paths can't end with "-"
			childBlockIndentation := computeItemBlockIndentation(i.curBlockStr(), i.eol)
			handleBasicBlock(i.newDescending(i.curBlockStart()+itemBlockStart, i.curBlockStart()+itemBlockEnd, blockIndentation), childBlockIndentation)
		}
	} else { // basic blocks
		blockIndentation := computeNormalBlockIndentation(i.curBlockStr(), i.eol)
		handleBasicBlock(i, blockIndentation)
	}
}

func handleBasicBlock(i traversingInfo, blockIndentation int) {
	matchedBlockPrefixes := regexp.
		MustCompile(i.eol+"["+indentationCharacters+"]{"+strconv.Itoa(blockIndentation)+"}"+i.unresolvedPath[0]).
		FindAllStringIndex(i.curBlockStr(), -1)
	if len(matchedBlockPrefixes) == 0 { //next block doesn't exist
		i.visitor(i)
		return
	}
	for _, prefixIndexes := range matchedBlockPrefixes {
		childBlockStart, childBlockEnd := computeChildBlockPosition(i.curBlockStr(), prefixIndexes, blockIndentation, i.eol)
		visitInsertionPlaces(i.newDescending(i.curBlockStart()+childBlockStart, i.curBlockStart()+childBlockEnd, blockIndentation))
	}
}

func computeNormalBlockIndentation(curBlock string, eol string) int {
	return computeBlockIndentation(curBlock, eol, eol+"["+spaceIndentationCharacter+"]*[^"+indentationCharacters+commentCharacters+"]{1}") //TODO convert indentationCharacters to regexp? for "- " is it the same
}

func computeMappingBlockIndentation(curBlock string, eol string) int {
	return computeBlockIndentation(curBlock, eol, eol+"["+spaceIndentationCharacter+"]*"+minusIndentationCharacter) //TODO convert indentationCharacters to regexp? for "- " is it the same
}

func computeItemBlockIndentation(curBlock string, eol string) int {
	return computeBlockIndentation(curBlock, eol, eol+"["+indentationCharacters+"]*[^"+indentationCharacters+commentCharacters+"]{1}") //TODO convert indentationCharacters to regexp? for "- " is it the same
}

func computeBlockIndentation(curBlock string, eol string, indentationRegExp string) int {
	indentation := regexp.MustCompile(indentationRegExp).FindString(curBlock)
	if indentation == "" { //block has no child blocks
		return -1
	}
	_, lastRuneSize := utf8.DecodeLastRuneInString(indentation)
	return len(indentation) - len(eol) - lastRuneSize //TODO convert byte length to rune length? for " " and "-" it is the same length
}

func computeChildBlockPosition(curBlock string, childBlockPrefixIndexes []int, blockIndentation int, eol string) (int, int) {
	return computeInnerBlockPosition(curBlock, childBlockPrefixIndexes, blockIndentation, eol, "[^"+indentationCharacters+commentCharacters+"]{1}")
}

func computeItemBlockPosition(curBlock string, itemBlockPrefixIndexes []int, blockIndentation int, eol string) (int, int) {
	return computeInnerBlockPosition(curBlock, itemBlockPrefixIndexes, blockIndentation, eol, minusIndentationCharacter)
}

func computeInnerBlockPosition(curBlock string, innerBlockPrefixIndexes []int, blockIndentation int, eol string, lastCharacterRegExp string) (int, int) {
	start := innerBlockPrefixIndexes[0] + len(eol)                                                                           //without eol from previous block
	nextSibling := eol + "[" + spaceIndentationCharacter + "]{" + strconv.Itoa(blockIndentation) + "}" + lastCharacterRegExp //TODO convert indentationCharacters to regexp? for "- " is it the same
	nextSiblingIdx := regexp.MustCompile(nextSibling).FindStringIndex(curBlock[innerBlockPrefixIndexes[0]+len(eol):])        // shifted search by len(eol) to not match start of block
	if nextSiblingIdx != nil {
		return start, innerBlockPrefixIndexes[0] + nextSiblingIdx[0] + len(eol) /*shifted sibling search*/ + len(eol) /*include block ending eol (matched by sibling search) */
	}
	return start, len(curBlock)
}

func detectEOLString(content string) (string, error) {
	for _, eol := range []string{"\r\n", "\n", "\r"} {
		if strings.Contains(content, eol) {
			return eol, nil
		}
	}
	return "", fmt.Errorf("can't detect end of line characters")
}

func isDeployment(document string) bool {
	return len(deploymentKind.FindString(document)) != 0
}

func isPod(document string) bool {
	return len(podKind.FindString(document)) != 0
}

func round(f float64) int {
	if f < -0.5 {
		return int(f - 0.5)
	}
	if f > 0.5 {
		return int(f + 0.5)
	}
	return 0
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

func loadSyms(t *testing.T) map[string]string {
	cmd := exec.Command("go", "tool", "nm", os.Args[0])
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("go tool nm %v: %v\n%s", os.Args[0], err, string(out))
	}
	syms := make(map[string]string)
	scanner := bufio.NewScanner(bytes.NewReader(out))
	for scanner.Scan() {
		f := strings.Fields(scanner.Text())
		if len(f) < 3 {
			continue
		}
		syms[f[2]] = f[0]
	}
	if err := scanner.Err(); err != nil {
		t.Fatalf("error reading symbols: %v", err)
	}
	return syms
}

func runAddr2Line(t *testing.T, exepath, addr string) (funcname, path, lineno string) {
	cmd := exec.Command(exepath, os.Args[0])
	cmd.Stdin = strings.NewReader(addr)
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("go tool addr2line %v: %v\n%s", os.Args[0], err, string(out))
	}
	f := strings.Split(string(out), "\n")
	if len(f) < 3 && f[2] == "" {
		t.Fatal("addr2line output must have 2 lines")
	}
	funcname = f[0]
	pathAndLineNo := f[1]
	f = strings.Split(pathAndLineNo, ":")
	if runtime.GOOS == "windows" {
		switch len(f) {
		case 2:
			return funcname, f[0], f[1]
		case 3:
			return funcname, f[0] + ":" + f[1], f[2]
		default:
			t.Fatalf("no line number found in %q", pathAndLineNo)
		}
	}
	if len(f) != 2 {
		t.Fatalf("no line number found in %q", pathAndLineNo)
	}
	return funcname, f[0], f[1]
}

const symName = "cmd/addr2line.TestAddr2Line"

func testAddr2Line(t *testing.T, exepath, addr string) {
	funcName, srcPath, srcLineNo := runAddr2Line(t, exepath, addr)
	if symName != funcName {
		t.Fatalf("expected function name %v; got %v", symName, funcName)
	}
	fi1, err := os.Stat("addr2line_test.go")
	if err != nil {
		t.Fatalf("Stat failed: %v", err)
	}
	fi2, err := os.Stat(srcPath)
	if err != nil {
		t.Fatalf("Stat failed: %v", err)
	}
	if !os.SameFile(fi1, fi2) {
		t.Fatalf("addr2line_test.go and %s are not same file", srcPath)
	}
	if srcLineNo != "94" {
		t.Fatalf("line number = %v; want 94", srcLineNo)
	}
}

// This is line 93. The test depends on that.
func TestAddr2Line(t *testing.T) {
	switch runtime.GOOS {
	case "nacl", "android":
		t.Skipf("skipping on %s", runtime.GOOS)
	}

	syms := loadSyms(t)

	tmpDir, err := ioutil.TempDir("", "TestAddr2Line")
	if err != nil {
		t.Fatal("TempDir failed: ", err)
	}
	defer os.RemoveAll(tmpDir)

	exepath := filepath.Join(tmpDir, "testaddr2line.exe")
	out, err := exec.Command("go", "build", "-o", exepath, "cmd/addr2line").CombinedOutput()
	if err != nil {
		t.Fatalf("go build -o %v cmd/addr2line: %v\n%s", exepath, err, string(out))
	}

	testAddr2Line(t, exepath, syms[symName])
	testAddr2Line(t, exepath, "0x"+syms[symName])
}
<commit_msg>cmd/addr2line: exclude Go tool test on darwin/arm<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"testing"
)

func loadSyms(t *testing.T) map[string]string {
	cmd := exec.Command("go", "tool", "nm", os.Args[0])
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("go tool nm %v: %v\n%s", os.Args[0], err, string(out))
	}
	syms := make(map[string]string)
	scanner := bufio.NewScanner(bytes.NewReader(out))
	for scanner.Scan() {
		f := strings.Fields(scanner.Text())
		if len(f) < 3 {
			continue
		}
		syms[f[2]] = f[0]
	}
	if err := scanner.Err(); err != nil {
		t.Fatalf("error reading symbols: %v", err)
	}
	return syms
}

func runAddr2Line(t *testing.T, exepath, addr string) (funcname, path, lineno string) {
	cmd := exec.Command(exepath, os.Args[0])
	cmd.Stdin = strings.NewReader(addr)
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("go tool addr2line %v: %v\n%s", os.Args[0], err, string(out))
	}
	f := strings.Split(string(out), "\n")
	if len(f) < 3 && f[2] == "" {
		t.Fatal("addr2line output must have 2 lines")
	}
	funcname = f[0]
	pathAndLineNo := f[1]
	f = strings.Split(pathAndLineNo, ":")
	if runtime.GOOS == "windows" {
		switch len(f) {
		case 2:
			return funcname, f[0], f[1]
		case 3:
			return funcname, f[0] + ":" + f[1], f[2]
		default:
			t.Fatalf("no line number found in %q", pathAndLineNo)
		}
	}
	if len(f) != 2 {
		t.Fatalf("no line number found in %q", pathAndLineNo)
	}
	return funcname, f[0], f[1]
}

const symName = "cmd/addr2line.TestAddr2Line"

func testAddr2Line(t *testing.T, exepath, addr string) {
	funcName, srcPath, srcLineNo := runAddr2Line(t, exepath, addr)
	if symName != funcName {
		t.Fatalf("expected function name %v; got %v", symName, funcName)
	}
	fi1, err := os.Stat("addr2line_test.go")
	if err != nil {
		t.Fatalf("Stat failed: %v", err)
	}
	fi2, err := os.Stat(srcPath)
	if err != nil {
		t.Fatalf("Stat failed: %v", err)
	}
	if !os.SameFile(fi1, fi2) {
		t.Fatalf("addr2line_test.go and %s are not same file", srcPath)
	}
	if srcLineNo != "94" {
		t.Fatalf("line number = %v; want 94", srcLineNo)
	}
}

// This is line 93. The test depends on that.
func TestAddr2Line(t *testing.T) {
	switch runtime.GOOS {
	case "nacl", "android":
		t.Skipf("skipping on %s", runtime.GOOS)
	case "darwin":
		if runtime.GOARCH == "arm" {
			t.Skipf("skipping on %s/%s", runtime.GOOS, runtime.GOARCH)
		}
	}

	syms := loadSyms(t)

	tmpDir, err := ioutil.TempDir("", "TestAddr2Line")
	if err != nil {
		t.Fatal("TempDir failed: ", err)
	}
	defer os.RemoveAll(tmpDir)

	exepath := filepath.Join(tmpDir, "testaddr2line.exe")
	out, err := exec.Command("go", "build", "-o", exepath, "cmd/addr2line").CombinedOutput()
	if err != nil {
		t.Fatalf("go build -o %v cmd/addr2line: %v\n%s", exepath, err, string(out))
	}

	testAddr2Line(t, exepath, syms[symName])
	testAddr2Line(t, exepath, "0x"+syms[symName])
}
<|endoftext|>


<commit_before>// Copyright 2013 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package handlers

import (
	"compress/flate"
	"compress/gzip"
	"io"
	"net/http"
	"strings"
)

type compressResponseWriter struct {
	io.Writer
	http.ResponseWriter
}

func (w *compressResponseWriter) Header() http.Header {
	return w.ResponseWriter.Header()
}

func (w *compressResponseWriter) Write(b []byte) (int, error) {
	h := w.ResponseWriter.Header()
	if h.Get("Content-Type") == "" {
		h.Set("Content-Type", http.DetectContentType(b))
	}

	return w.Writer.Write(b)
}

func CompressHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	L:
		for _, enc := range strings.Split(r.Header.Get("Accept-Encoding"), ",") {
			switch strings.TrimSpace(enc) {
			case "gzip":
				w.Header().Set("Content-Encoding", "gzip")
				w.Header().Add("Vary", "Accept-Encoding")

				gw := gzip.NewWriter(w)
				defer gw.Close()

				w = &compressResponseWriter{
					Writer:         gw,
					ResponseWriter: w,
				}
				break L
			case "deflate":
				w.Header().Set("Content-Encoding", "deflate")
				w.Header().Add("Vary", "Accept-Encoding")

				fw, _ := flate.NewWriter(w, flate.DefaultCompression)
				defer fw.Close()

				w = &compressResponseWriter{
					Writer:         fw,
					ResponseWriter: w,
				}
				break L
			}
		}

		h.ServeHTTP(w, r)
	})
}
<commit_msg>Implement Hijacker too for Websocket to work<commit_after>// Copyright 2013 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package handlers

import (
	"compress/flate"
	"compress/gzip"
	"io"
	"net/http"
	"strings"
)

type compressResponseWriter struct {
	io.Writer
	http.ResponseWriter
	http.Hijacker
}

func (w *compressResponseWriter) Header() http.Header {
	return w.ResponseWriter.Header()
}

func (w *compressResponseWriter) Write(b []byte) (int, error) {
	h := w.ResponseWriter.Header()
	if h.Get("Content-Type") == "" {
		h.Set("Content-Type", http.DetectContentType(b))
	}

	return w.Writer.Write(b)
}

func CompressHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	L:
		for _, enc := range strings.Split(r.Header.Get("Accept-Encoding"), ",") {
			switch strings.TrimSpace(enc) {
			case "gzip":
				w.Header().Set("Content-Encoding", "gzip")
				w.Header().Add("Vary", "Accept-Encoding")

				gw := gzip.NewWriter(w)
				defer gw.Close()

				w = &compressResponseWriter{
					Writer:         gw,
					ResponseWriter: w,
					Hijacker:       w.(http.Hijacker),
				}
				break L
			case "deflate":
				w.Header().Set("Content-Encoding", "deflate")
				w.Header().Add("Vary", "Accept-Encoding")

				fw, _ := flate.NewWriter(w, flate.DefaultCompression)
				defer fw.Close()

				w = &compressResponseWriter{
					Writer:         fw,
					ResponseWriter: w,
					Hijacker:       w.(http.Hijacker),
				}
				break L
			}
		}

		h.ServeHTTP(w, r)
	})
}
<|endoftext|>


<commit_before>package task_bbs

import (
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/bbserrors"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/storeadapter"
	"github.com/pivotal-golang/lager"
)

// The stager calls this when it wants to desire a payload
// stagerTaskBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?)
// If this fails, the stager should bail and run its "this-failed-to-stage" routine
func (bbs *TaskBBS) DesireTask(logger lager.Logger, task models.Task) error {
	taskLogger := logger.WithData(lager.Data{"task-guid": task.TaskGuid})

	taskLogger.Info("starting")
	defer taskLogger.Info("finished")

	err := task.Validate()
	if err != nil {
		return err
	}
	task.State = models.TaskStatePending

	if task.CreatedAt == 0 {
		task.CreatedAt = bbs.clock.Now().UnixNano()
	}

	task.UpdatedAt = bbs.clock.Now().UnixNano()

	value, err := models.ToJSON(task)
	if err != nil {
		return err
	}

	taskLogger.Debug("persisting-task")
	err = bbs.store.Create(storeadapter.StoreNode{
		Key:   shared.TaskSchemaPath(task.TaskGuid),
		Value: value,
	})
	if err != nil {
		taskLogger.Error("failed-persisting-task", err)
		return shared.ConvertStoreError(err)
	}
	taskLogger.Debug("succeeded-persisting-task")

	taskLogger.Debug("requesting-task-auction")
	err = bbs.requestTaskAuctions(taskLogger, []models.Task{task})
	if err != nil {
		taskLogger.Error("failed-requesting-task-auction", err)
		// The creation succeeded, the auction request error can be dropped
	} else {
		taskLogger.Debug("succeeded-requesting-task-auction")
	}

	return nil
}

// The cell calls this when it is about to run the task in the allocated container
// stagerTaskBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?)
// If this fails, the cell should assume that someone else will run it and should clean up and bail
func (bbs *TaskBBS) StartTask(logger lager.Logger, taskGuid string, cellID string) (bool, error) {
	logger = logger.Session("start-task", lager.Data{"task-guid": taskGuid, "cell-id": cellID})

	logger.Info("starting")
	defer logger.Info("finished")

	logger.Info("getting-task")
	task, index, err := bbs.getTask(taskGuid)
	if err != nil {
		logger.Error("failed-getting-task", err)
		return false, err
	}
	logger.Info("succeeded-getting-task")

	if task.State == models.TaskStateRunning && task.CellID == cellID {
		logger.Info("task-already-running")
		return false, nil
	}

	err = validateStateTransition(task.State, models.TaskStateRunning)
	if err != nil {
		logger.Error("invalid-state-transition", err)
		return false, err
	}

	task.UpdatedAt = bbs.clock.Now().UnixNano()
	task.State = models.TaskStateRunning
	task.CellID = cellID

	value, err := models.ToJSON(task)
	if err != nil {
		logger.Error("failed-converting-to-json", err)
		return false, err
	}

	logger.Info("persisting-task")
	err = bbs.store.CompareAndSwapByIndex(index, storeadapter.StoreNode{
		Key:   shared.TaskSchemaPath(taskGuid),
		Value: value,
	})
	if err != nil {
		logger.Error("failed-persisting-task", err)
		return false, shared.ConvertStoreError(err)
	}
	logger.Info("succeeded-persisting-task")

	return true, nil
}

// The cell calls this when the user requested to cancel the task
// stagerTaskBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?)
// Will fail if the task has already been cancelled or completed normally
func (bbs *TaskBBS) CancelTask(logger lager.Logger, taskGuid string) error {
	logger = logger.Session("cancel-task", lager.Data{"task-guid": taskGuid})

	logger.Info("starting")
	defer logger.Info("finished")

	logger.Info("getting-task")
	task, index, err := bbs.getTask(taskGuid)
	if err != nil {
		logger.Error("failed-getting-task", err)
		return err
	}
	logger.Info("succeeded-getting-task")

	if task.State == models.TaskStateResolving || task.State == models.TaskStateCompleted {
		err = bbserrors.NewTaskStateTransitionError(task.State, models.TaskStateCompleted)
		logger.Error("invalid-state-transition", err)
		return err
	}

	logger.Info("completing-task")
	err = bbs.completeTask(logger, task, index, true, "task was cancelled", "")
	if err != nil {
		logger.Error("failed-completing-task", err)
		return err
	}
	logger.Info("succeeded-completing-task")

	if task.CellID == "" {
		return nil
	}

	logger.Info("getting-cell-info")
	cellPresence, err := bbs.services.CellById(task.CellID)
	if err != nil {
		logger.Error("failed-getting-cell-info", err)
		return nil
	}
	logger.Info("succeeded-getting-cell-info")

	logger.Info("cell-client-cancelling-task")
	err = bbs.cellClient.CancelTask(cellPresence.RepAddress, task.TaskGuid)
	if err != nil {
		logger.Error("cell-client-failed-cancelling-task", err)
		return nil
	}
	logger.Info("cell-client-succeeded-cancelling-task")

	return nil
}

func (bbs *TaskBBS) FailTask(logger lager.Logger, taskGuid string, failureReason string) error {
	logger = logger.Session("fail-task", lager.Data{"task-guid": taskGuid})

	logger.Info("starting")
	defer logger.Info("finished")

	logger.Info("getting-task")
	task, index, err := bbs.getTask(taskGuid)
	if err != nil {
		logger.Error("failed-getting-task", err)
		return err
	}
	logger.Info("succeeded-getting-task")

	if task.State == models.TaskStateResolving || task.State == models.TaskStateCompleted {
		err = bbserrors.NewTaskStateTransitionError(task.State, models.TaskStateCompleted)
		logger.Error("invalid-state-transition", err)
		return err
	}

	return bbs.completeTask(logger, task, index, true, failureReason, "")
}

// The cell calls this when it has finished running the task (be it success or failure)
// stagerTaskBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?)
// This really really shouldn't fail.  If it does, blog about it and walk away. If it failed in a
// consistent way (i.e. key already exists), there's probably a flaw in our design.
func (bbs *TaskBBS) CompleteTask(logger lager.Logger, taskGuid string, cellID string, failed bool, failureReason string, result string) error {
	logger = logger.Session("complete-task", lager.Data{"task-guid": taskGuid, "cell-id": cellID})

	logger.Info("starting")
	defer logger.Info("finished")

	logger.Info("getting-task")
	task, index, err := bbs.getTask(taskGuid)
	if err != nil {
		logger.Error("failed-getting-task", err)
		return err
	}
	logger.Info("succeeded-getting-task")

	if task.State == models.TaskStateRunning && task.CellID != cellID {
		err = bbserrors.ErrTaskRunningOnDifferentCell
		logger.Error("invalid-cell-id", err)
		return err
	}

	err = validateStateTransition(task.State, models.TaskStateCompleted)
	if err != nil {
		logger.Error("invalid-state-transition", err)
		return err
	}

	return bbs.completeTask(logger, task, index, failed, failureReason, result)
}

func (bbs *TaskBBS) completeTask(logger lager.Logger, task models.Task, index uint64, failed bool, failureReason string, result string) error {
	task = bbs.markTaskCompleted(task, failed, failureReason, result)

	value, err := models.ToJSON(task)
	if err != nil {
		return err
	}

	logger.Info("persisting-task")
	err = bbs.store.CompareAndSwapByIndex(index, storeadapter.StoreNode{
		Key:   shared.TaskSchemaPath(task.TaskGuid),
		Value: value,
	})
	if err != nil {
		logger.Error("failed-persisting-task", err)
		return shared.ConvertStoreError(err)
	}
	logger.Info("succeded-persisting-task")

	if task.CompletionCallbackURL == nil {
		return nil
	}

	logger.Info("task-client-completing-task")
	err = bbs.taskClient.CompleteTasks(bbs.receptorTaskHandlerURL, []models.Task{task})
	if err != nil {
		logger.Error("task-client-failed-completing-task", err)
		return nil
	}
	logger.Info("task-client-succeeded-completing-task")

	return nil
}

// The stager calls this when it wants to claim a completed task.  This ensures that only one
// stager ever attempts to handle a completed task
func (bbs *TaskBBS) ResolvingTask(logger lager.Logger, taskGuid string) error {
	logger = logger.Session("resolving-task", lager.Data{"task-guid": taskGuid})

	logger.Info("starting")
	defer logger.Info("finished")

	logger.Info("getting-task")
	task, index, err := bbs.getTask(taskGuid)
	if err != nil {
		logger.Error("failed-getting-task", err)
		return err
	}
	logger.Info("succeeded-getting-task")

	err = validateStateTransition(task.State, models.TaskStateResolving)
	if err != nil {
		logger.Error("invalid-state-transition", err)
		return err
	}

	task.UpdatedAt = bbs.clock.Now().UnixNano()
	task.State = models.TaskStateResolving

	value, err := models.ToJSON(task)
	if err != nil {
		return err
	}

	return shared.ConvertStoreError(bbs.store.CompareAndSwapByIndex(index, storeadapter.StoreNode{
		Key:   shared.TaskSchemaPath(taskGuid),
		Value: value,
	}))
}

// The stager calls this when it wants to signal that it has received a completion and is handling it
// stagerTaskBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?)
// If this fails, the stager should assume that someone else is handling the completion and should bail
func (bbs *TaskBBS) ResolveTask(logger lager.Logger, taskGuid string) error {
	logger = logger.Session("resolve-task", lager.Data{"task-guid": taskGuid})

	logger.Info("starting")
	defer logger.Info("finished")

	logger.Info("getting-task")
	task, _, err := bbs.getTask(taskGuid)
	if err != nil {
		logger.Error("failed-getting-task", err)
		return err
	}
	logger.Info("succeeded-getting-task")

	err = validateCanDelete(task.State)
	if err != nil {
		logger.Error("invalid-state-transition", err)
		return err
	}

	return shared.ConvertStoreError(bbs.store.Delete(shared.TaskSchemaPath(taskGuid)))
}

func validateStateTransition(from, to models.TaskState) error {
	if (from == models.TaskStatePending && to == models.TaskStateRunning) ||
		(from == models.TaskStateRunning && to == models.TaskStateCompleted) ||
		(from == models.TaskStateCompleted && to == models.TaskStateResolving) {
		return nil
	} else {
		return bbserrors.NewTaskStateTransitionError(from, to)
	}
}

func validateCanDelete(from models.TaskState) error {
	if from != models.TaskStateResolving {
		return bbserrors.NewTaskCannotBeResolvedError(from)
	} else {
		return nil
	}
}

func (bbs *TaskBBS) requestTaskAuctions(logger lager.Logger, tasks []models.Task) error {
	auctioneerAddress, err := bbs.services.AuctioneerAddress()
	if err != nil {
		return err
	}
	logger.Debug("did-fetch-auctioneer-address")

	err = bbs.auctioneerClient.RequestTaskAuctions(auctioneerAddress, tasks)
	if err != nil {
		return err
	}
	logger.Debug("did-request-task-auctions")

	return nil
}
<commit_msg>Add desire-task label to logger session<commit_after>package task_bbs

import (
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/bbserrors"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/storeadapter"
	"github.com/pivotal-golang/lager"
)

// The stager calls this when it wants to desire a payload
// stagerTaskBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?)
// If this fails, the stager should bail and run its "this-failed-to-stage" routine
func (bbs *TaskBBS) DesireTask(logger lager.Logger, task models.Task) error {
	taskLogger := logger.Session("desire-task", lager.Data{"task-guid": task.TaskGuid})

	taskLogger.Info("starting")
	defer taskLogger.Info("finished")

	err := task.Validate()
	if err != nil {
		return err
	}
	task.State = models.TaskStatePending

	if task.CreatedAt == 0 {
		task.CreatedAt = bbs.clock.Now().UnixNano()
	}

	task.UpdatedAt = bbs.clock.Now().UnixNano()

	value, err := models.ToJSON(task)
	if err != nil {
		return err
	}

	taskLogger.Debug("persisting-task")
	err = bbs.store.Create(storeadapter.StoreNode{
		Key:   shared.TaskSchemaPath(task.TaskGuid),
		Value: value,
	})
	if err != nil {
		taskLogger.Error("failed-persisting-task", err)
		return shared.ConvertStoreError(err)
	}
	taskLogger.Debug("succeeded-persisting-task")

	taskLogger.Debug("requesting-task-auction")
	err = bbs.requestTaskAuctions(taskLogger, []models.Task{task})
	if err != nil {
		taskLogger.Error("failed-requesting-task-auction", err)
		// The creation succeeded, the auction request error can be dropped
	} else {
		taskLogger.Debug("succeeded-requesting-task-auction")
	}

	return nil
}

// The cell calls this when it is about to run the task in the allocated container
// stagerTaskBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?)
// If this fails, the cell should assume that someone else will run it and should clean up and bail
func (bbs *TaskBBS) StartTask(logger lager.Logger, taskGuid string, cellID string) (bool, error) {
	logger = logger.Session("start-task", lager.Data{"task-guid": taskGuid, "cell-id": cellID})

	logger.Info("starting")
	defer logger.Info("finished")

	logger.Info("getting-task")
	task, index, err := bbs.getTask(taskGuid)
	if err != nil {
		logger.Error("failed-getting-task", err)
		return false, err
	}
	logger.Info("succeeded-getting-task")

	if task.State == models.TaskStateRunning && task.CellID == cellID {
		logger.Info("task-already-running")
		return false, nil
	}

	err = validateStateTransition(task.State, models.TaskStateRunning)
	if err != nil {
		logger.Error("invalid-state-transition", err)
		return false, err
	}

	task.UpdatedAt = bbs.clock.Now().UnixNano()
	task.State = models.TaskStateRunning
	task.CellID = cellID

	value, err := models.ToJSON(task)
	if err != nil {
		logger.Error("failed-converting-to-json", err)
		return false, err
	}

	logger.Info("persisting-task")
	err = bbs.store.CompareAndSwapByIndex(index, storeadapter.StoreNode{
		Key:   shared.TaskSchemaPath(taskGuid),
		Value: value,
	})
	if err != nil {
		logger.Error("failed-persisting-task", err)
		return false, shared.ConvertStoreError(err)
	}
	logger.Info("succeeded-persisting-task")

	return true, nil
}

// The cell calls this when the user requested to cancel the task
// stagerTaskBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?)
// Will fail if the task has already been cancelled or completed normally
func (bbs *TaskBBS) CancelTask(logger lager.Logger, taskGuid string) error {
	logger = logger.Session("cancel-task", lager.Data{"task-guid": taskGuid})

	logger.Info("starting")
	defer logger.Info("finished")

	logger.Info("getting-task")
	task, index, err := bbs.getTask(taskGuid)
	if err != nil {
		logger.Error("failed-getting-task", err)
		return err
	}
	logger.Info("succeeded-getting-task")

	if task.State == models.TaskStateResolving || task.State == models.TaskStateCompleted {
		err = bbserrors.NewTaskStateTransitionError(task.State, models.TaskStateCompleted)
		logger.Error("invalid-state-transition", err)
		return err
	}

	logger.Info("completing-task")
	err = bbs.completeTask(logger, task, index, true, "task was cancelled", "")
	if err != nil {
		logger.Error("failed-completing-task", err)
		return err
	}
	logger.Info("succeeded-completing-task")

	if task.CellID == "" {
		return nil
	}

	logger.Info("getting-cell-info")
	cellPresence, err := bbs.services.CellById(task.CellID)
	if err != nil {
		logger.Error("failed-getting-cell-info", err)
		return nil
	}
	logger.Info("succeeded-getting-cell-info")

	logger.Info("cell-client-cancelling-task")
	err = bbs.cellClient.CancelTask(cellPresence.RepAddress, task.TaskGuid)
	if err != nil {
		logger.Error("cell-client-failed-cancelling-task", err)
		return nil
	}
	logger.Info("cell-client-succeeded-cancelling-task")

	return nil
}

func (bbs *TaskBBS) FailTask(logger lager.Logger, taskGuid string, failureReason string) error {
	logger = logger.Session("fail-task", lager.Data{"task-guid": taskGuid})

	logger.Info("starting")
	defer logger.Info("finished")

	logger.Info("getting-task")
	task, index, err := bbs.getTask(taskGuid)
	if err != nil {
		logger.Error("failed-getting-task", err)
		return err
	}
	logger.Info("succeeded-getting-task")

	if task.State == models.TaskStateResolving || task.State == models.TaskStateCompleted {
		err = bbserrors.NewTaskStateTransitionError(task.State, models.TaskStateCompleted)
		logger.Error("invalid-state-transition", err)
		return err
	}

	return bbs.completeTask(logger, task, index, true, failureReason, "")
}

// The cell calls this when it has finished running the task (be it success or failure)
// stagerTaskBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?)
// This really really shouldn't fail.  If it does, blog about it and walk away. If it failed in a
// consistent way (i.e. key already exists), there's probably a flaw in our design.
func (bbs *TaskBBS) CompleteTask(logger lager.Logger, taskGuid string, cellID string, failed bool, failureReason string, result string) error {
	logger = logger.Session("complete-task", lager.Data{"task-guid": taskGuid, "cell-id": cellID})

	logger.Info("starting")
	defer logger.Info("finished")

	logger.Info("getting-task")
	task, index, err := bbs.getTask(taskGuid)
	if err != nil {
		logger.Error("failed-getting-task", err)
		return err
	}
	logger.Info("succeeded-getting-task")

	if task.State == models.TaskStateRunning && task.CellID != cellID {
		err = bbserrors.ErrTaskRunningOnDifferentCell
		logger.Error("invalid-cell-id", err)
		return err
	}

	err = validateStateTransition(task.State, models.TaskStateCompleted)
	if err != nil {
		logger.Error("invalid-state-transition", err)
		return err
	}

	return bbs.completeTask(logger, task, index, failed, failureReason, result)
}

func (bbs *TaskBBS) completeTask(logger lager.Logger, task models.Task, index uint64, failed bool, failureReason string, result string) error {
	task = bbs.markTaskCompleted(task, failed, failureReason, result)

	value, err := models.ToJSON(task)
	if err != nil {
		return err
	}

	logger.Info("persisting-task")
	err = bbs.store.CompareAndSwapByIndex(index, storeadapter.StoreNode{
		Key:   shared.TaskSchemaPath(task.TaskGuid),
		Value: value,
	})
	if err != nil {
		logger.Error("failed-persisting-task", err)
		return shared.ConvertStoreError(err)
	}
	logger.Info("succeded-persisting-task")

	if task.CompletionCallbackURL == nil {
		return nil
	}

	logger.Info("task-client-completing-task")
	err = bbs.taskClient.CompleteTasks(bbs.receptorTaskHandlerURL, []models.Task{task})
	if err != nil {
		logger.Error("task-client-failed-completing-task", err)
		return nil
	}
	logger.Info("task-client-succeeded-completing-task")

	return nil
}

// The stager calls this when it wants to claim a completed task.  This ensures that only one
// stager ever attempts to handle a completed task
func (bbs *TaskBBS) ResolvingTask(logger lager.Logger, taskGuid string) error {
	logger = logger.Session("resolving-task", lager.Data{"task-guid": taskGuid})

	logger.Info("starting")
	defer logger.Info("finished")

	logger.Info("getting-task")
	task, index, err := bbs.getTask(taskGuid)
	if err != nil {
		logger.Error("failed-getting-task", err)
		return err
	}
	logger.Info("succeeded-getting-task")

	err = validateStateTransition(task.State, models.TaskStateResolving)
	if err != nil {
		logger.Error("invalid-state-transition", err)
		return err
	}

	task.UpdatedAt = bbs.clock.Now().UnixNano()
	task.State = models.TaskStateResolving

	value, err := models.ToJSON(task)
	if err != nil {
		return err
	}

	return shared.ConvertStoreError(bbs.store.CompareAndSwapByIndex(index, storeadapter.StoreNode{
		Key:   shared.TaskSchemaPath(taskGuid),
		Value: value,
	}))
}

// The stager calls this when it wants to signal that it has received a completion and is handling it
// stagerTaskBBS will retry this repeatedly if it gets a StoreTimeout error (up to N seconds?)
// If this fails, the stager should assume that someone else is handling the completion and should bail
func (bbs *TaskBBS) ResolveTask(logger lager.Logger, taskGuid string) error {
	logger = logger.Session("resolve-task", lager.Data{"task-guid": taskGuid})

	logger.Info("starting")
	defer logger.Info("finished")

	logger.Info("getting-task")
	task, _, err := bbs.getTask(taskGuid)
	if err != nil {
		logger.Error("failed-getting-task", err)
		return err
	}
	logger.Info("succeeded-getting-task")

	err = validateCanDelete(task.State)
	if err != nil {
		logger.Error("invalid-state-transition", err)
		return err
	}

	return shared.ConvertStoreError(bbs.store.Delete(shared.TaskSchemaPath(taskGuid)))
}

func validateStateTransition(from, to models.TaskState) error {
	if (from == models.TaskStatePending && to == models.TaskStateRunning) ||
		(from == models.TaskStateRunning && to == models.TaskStateCompleted) ||
		(from == models.TaskStateCompleted && to == models.TaskStateResolving) {
		return nil
	} else {
		return bbserrors.NewTaskStateTransitionError(from, to)
	}
}

func validateCanDelete(from models.TaskState) error {
	if from != models.TaskStateResolving {
		return bbserrors.NewTaskCannotBeResolvedError(from)
	} else {
		return nil
	}
}

func (bbs *TaskBBS) requestTaskAuctions(logger lager.Logger, tasks []models.Task) error {
	auctioneerAddress, err := bbs.services.AuctioneerAddress()
	if err != nil {
		return err
	}
	logger.Debug("did-fetch-auctioneer-address")

	err = bbs.auctioneerClient.RequestTaskAuctions(auctioneerAddress, tasks)
	if err != nil {
		return err
	}
	logger.Debug("did-request-task-auctions")

	return nil
}
<|endoftext|>


<commit_before>
package main

import (
	"fmt"
	"time"
	"math/rand"
)

type Result string
type Search func(query string) Result

var (
	Web = fakeSearch("web")
	Image = fakeSearch("image")
	Video = fakeSearch("video")
)

func fakeSearch(kind string) func(query string) Result {
	return func(query string) Result {
		time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
		return Result(fmt.Sprintf("%s result for %q\n", kind, query))
	}
}

func Google(query string) (results []Result) {
	c := make(chan Result)

	go func() { c <- Web(query) }()
	go func() { c <- Image(query) }()
	go func() { c <- Video(query) }()

	timeout := time.After(80 * time.Millisecond)

	for i := 0; i < 3; i++ {
		select {
		case r := <- c:
			results = append(results, r)
		case <- timeout:
			fmt.Println("timed out")
			return
		}
	}

	return
}

func main() {
	rand.Seed(time.Now().UnixNano())
	start := time.Now()
	results := Google("golang")
	elapsed := time.Since(start)
	fmt.Println(results)
	fmt.Println(elapsed)
}
<commit_msg>Added replication on Google search example<commit_after>
package main

import (
	"fmt"
	"time"
	"math/rand"
)

type Result string
type Search func(query string) Result

var (
	Web = []Search{
		fakeSearch("web1.google.com"),
		fakeSearch("web2.google.com"),
		fakeSearch("web3.google.com")}

	Image = []Search{
		fakeSearch("image1.google.com"),
		fakeSearch("image2.google.com"),
		fakeSearch("image3.google.com")}

	Video = []Search{
		fakeSearch("video1.google.com"),
		fakeSearch("video2.google.com"),
		fakeSearch("video3.google.com")}
)

func fakeSearch(kind string) func(query string) Result {
	return func(query string) Result {
		time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
		return Result(fmt.Sprintf("%s result for %q\n", kind, query))
	}
}

func First(query string, replicas ...Search) Result {
	c := make(chan Result)

	for _, r := range replicas {
		go func() { c <- r(query) }()
	}

	return <- c
}

func Google(query string) (results []Result) {
	c := make(chan Result)

	go func() { c <- First(query, Web...) }()
	go func() { c <- First(query, Image...) }()
	go func() { c <- First(query, Video...) }()

	timeout := time.After(80 * time.Millisecond)

	for i := 0; i < 3; i++ {
		select {
		case r := <- c:
			results = append(results, r)
		case <- timeout:
			fmt.Println("timed out")
			return
		}
	}

	return
}

func main() {
	rand.Seed(time.Now().UnixNano())
	start := time.Now()
	results := Google("golang")
	elapsed := time.Since(start)
	fmt.Println(results)
	fmt.Println(elapsed)
}
<|endoftext|>