<commit_before>package deploymentconfig

import (
	"fmt"
	"strconv"

	"github.com/golang/glog"

	kapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/client/record"
	kclient "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/runtime"
	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
	"k8s.io/kubernetes/pkg/util/workqueue"

	osclient "github.com/openshift/origin/pkg/client"
	oscache "github.com/openshift/origin/pkg/client/cache"
	deployapi "github.com/openshift/origin/pkg/deploy/api"
	deployutil "github.com/openshift/origin/pkg/deploy/util"
)

// fatalError is an error which can't be retried.
type fatalError string

func (e fatalError) Error() string {
	return fmt.Sprintf("fatal error handling deployment config: %s", string(e))
}

// DeploymentConfigController is responsible for creating a new deployment
// when:
//
//    1. The config version is > 0 and,
//    2. No deployment for the version exists.
//
// The controller reconciles deployments with the replica count specified on
// the config. The active deployment (that is, the latest successful
// deployment) will always be scaled to the config replica count. All other
// deployments will be scaled to zero.
//
// If a new version is observed for which no deployment exists, any running
// deployments will be cancelled. The controller will not attempt to scale
// running deployments.
type DeploymentConfigController struct {
	// dn provides access to deploymentconfigs.
	dn osclient.DeploymentConfigsNamespacer
	// rn provides access to replication controllers.
	rn kclient.ReplicationControllersNamespacer

	// queue contains deployment configs that need to be synced.
	queue workqueue.RateLimitingInterface

	// dcStore provides a local cache for deployment configs.
	dcStore oscache.StoreToDeploymentConfigLister
	// rcStore provides a local cache for replication controllers.
	rcStore cache.StoreToReplicationControllerLister
	// dcStoreSynced makes sure the dc store is synced before reconcling any deployment config.
	dcStoreSynced func() bool
	// rcStoreSynced makes sure the rc store is synced before reconcling any deployment config.
	rcStoreSynced func() bool

	// codec is used to build deployments from configs.
	codec runtime.Codec
	// recorder is used to record events.
	recorder record.EventRecorder
}

// Handle implements the loop that processes deployment configs. Since this controller started
// using caches, the provided config MUST be deep-copied beforehand (see work() in factory.go).
func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error {
	// There's nothing to reconcile until the version is nonzero or when the
	// deployment config has been marked for deletion.
	if config.Status.LatestVersion == 0 || config.DeletionTimestamp != nil {
		return c.updateStatus(config)
	}

	// Find all deployments owned by the deployment config.
	selector := deployutil.ConfigSelector(config.Name)
	existingDeployments, err := c.rcStore.ReplicationControllers(config.Namespace).List(selector)
	if err != nil {
		return err
	}

	latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments)
	// If the latest deployment doesn't exist yet, cancel any running
	// deployments to allow them to be superceded by the new config version.
	awaitingCancellations := false
	if !latestIsDeployed {
		for _, deployment := range existingDeployments {
			// Skip deployments with an outcome.
			if deployutil.IsTerminatedDeployment(&deployment) {
				continue
			}
			// Cancel running deployments.
			awaitingCancellations = true
			if !deployutil.IsDeploymentCancelled(&deployment) {
				deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue
				deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists
				_, err := c.rn.ReplicationControllers(deployment.Namespace).Update(&deployment)
				if err != nil {
					c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCancellationFailed", "Failed to cancel deployment %q superceded by version %d: %s", deployment.Name, config.Status.LatestVersion, err)
				} else {
					c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCancelled", "Cancelled deployment %q superceded by version %d", deployment.Name, config.Status.LatestVersion)
				}
			}
		}
	}
	// Wait for deployment cancellations before reconciling or creating a new
	// deployment to avoid competing with existing deployment processes.
	if awaitingCancellations {
		c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentAwaitingCancellation", "Deployment of version %d awaiting cancellation of older running deployments", config.Status.LatestVersion)
		return fmt.Errorf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config))
	}
	// If the latest deployment already exists, reconcile existing deployments
	// and return early.
	if latestIsDeployed {
		// If the latest deployment is still running, try again later. We don't
		// want to compete with the deployer.
		if !deployutil.IsTerminatedDeployment(latestDeployment) {
			return c.updateStatus(config)
		}
		return c.reconcileDeployments(existingDeployments, config)
	}
	// If the config is paused we shouldn't create new deployments for it.
	// TODO: Make sure cleanup policy will work for paused configs.
	if config.Spec.Paused {
		return c.updateStatus(config)
	}
	// No deployments are running and the latest deployment doesn't exist, so
	// create the new deployment.
	deployment, err := deployutil.MakeDeployment(config, c.codec)
	if err != nil {
		return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err))
	}
	created, err := c.rn.ReplicationControllers(config.Namespace).Create(deployment)
	if err != nil {
		// If the deployment was already created, just move on. The cache could be
		// stale, or another process could have already handled this update.
		if errors.IsAlreadyExists(err) {
			return c.updateStatus(config)
		}
		c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCreationFailed", "Couldn't deploy version %d: %s", config.Status.LatestVersion, err)
		return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)
	}
	c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCreated", "Created new deployment %q for version %d", created.Name, config.Status.LatestVersion)

	return c.updateStatus(config)
}

// reconcileDeployments reconciles existing deployment replica counts which
// could have diverged outside the deployment process (e.g. due to auto or
// manual scaling, or partial deployments). The active deployment is the last
// successful deployment, not necessarily the latest in terms of the config
// version. The active deployment replica count should follow the config, and
// all other deployments should be scaled to zero.
//
// Previously, scaling behavior was that the config replica count was used
// only for initial deployments and the active deployment had to be scaled up
// directly. To continue supporting that old behavior we must detect when the
// deployment has been directly manipulated, and if so, preserve the directly
// updated value and sync the config with the deployment.
func (c *DeploymentConfigController) reconcileDeployments(existingDeployments []kapi.ReplicationController, config *deployapi.DeploymentConfig) error {
	latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments)
	if !latestIsDeployed {
		// We shouldn't be reconciling if the latest deployment hasn't been
		// created; this is enforced on the calling side, but double checking
		// can't hurt.
		return nil
	}
	activeDeployment := deployutil.ActiveDeployment(config, existingDeployments)
	// Compute the replica count for the active deployment (even if the active
	// deployment doesn't exist). The active replica count is the value that
	// should be assigned to the config, to allow the replica propagation to
	// flow downward from the config.
	//
	// By default we'll assume the config replicas should be used to update the
	// active deployment except in special cases (like first sync or externally
	// updated deployments.)
	activeReplicas := config.Spec.Replicas
	source := "the deploymentConfig itself (no change)"

	activeDeploymentExists := activeDeployment != nil
	activeDeploymentIsLatest := activeDeploymentExists && activeDeployment.Name == latestDeployment.Name
	latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment)

	switch {
	case activeDeploymentExists && activeDeploymentIsLatest:
		// The active/latest deployment follows the config unless this is its first
		// sync or if an external change to the deployment replicas is detected.
		lastActiveReplicas, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment)
		if !hasLastActiveReplicas || lastActiveReplicas != activeDeployment.Spec.Replicas {
			activeReplicas = activeDeployment.Spec.Replicas
			source = fmt.Sprintf("the latest/active deployment %q which was scaled directly or has not previously been synced", deployutil.LabelForDeployment(activeDeployment))
		}
	case activeDeploymentExists && !activeDeploymentIsLatest:
		// The active/non-latest deployment follows the config if it was
		// previously synced; if this is the first sync, infer what the config
		// value should be based on either the latest desired or whatever the
		// deployment is currently scaled to.
		_, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment)
		if hasLastActiveReplicas {
			break
		}
		if latestHasDesiredReplicas {
			activeReplicas = latestDesiredReplicas
			source = fmt.Sprintf("the desired replicas of latest deployment %q which has not been previously synced", deployutil.LabelForDeployment(latestDeployment))
		} else if activeDeployment.Spec.Replicas > 0 {
			activeReplicas = activeDeployment.Spec.Replicas
			source = fmt.Sprintf("the active deployment %q which has not been previously synced", deployutil.LabelForDeployment(activeDeployment))
		}
	case !activeDeploymentExists && latestHasDesiredReplicas:
		// If there's no active deployment, use the latest desired, if available.
		activeReplicas = latestDesiredReplicas
		source = fmt.Sprintf("the desired replicas of latest deployment %q with no active deployment", deployutil.LabelForDeployment(latestDeployment))
	}

	// Bring the config in sync with the deployment. Once we know the config
	// accurately represents the desired replica count of the active deployment,
	// we can safely reconcile deployments.
	//
	// If the deployment config is test, never update the deployment config based
	// on deployments, since test behavior overrides user scaling.
	switch {
	case config.Spec.Replicas == activeReplicas:
	case config.Spec.Test:
		glog.V(4).Infof("Detected changed replicas for test deploymentConfig %q, ignoring that change", deployutil.LabelForDeploymentConfig(config))
	default:
		oldReplicas := config.Spec.Replicas
		config.Spec.Replicas = activeReplicas
		var err error
		config, err = c.dn.DeploymentConfigs(config.Namespace).Update(config)
		if err != nil {
			return err
		}
		glog.V(4).Infof("Synced deploymentConfig %q replicas from %d to %d based on %s", deployutil.LabelForDeploymentConfig(config), oldReplicas, activeReplicas, source)
	}

	// Reconcile deployments. The active deployment follows the config, and all
	// other deployments should be scaled to zero.
	for _, deployment := range existingDeployments {
		isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name

		oldReplicaCount := deployment.Spec.Replicas
		newReplicaCount := int32(0)
		if isActiveDeployment {
			newReplicaCount = activeReplicas
		}
		if config.Spec.Test {
			glog.V(4).Infof("Deployment config %q is test and deployment %q will be scaled down", deployutil.LabelForDeploymentConfig(config), deployutil.LabelForDeployment(&deployment))
			newReplicaCount = 0
		}
		lastReplicas, hasLastReplicas := deployutil.DeploymentReplicas(&deployment)
		// Only update if necessary.
		if !hasLastReplicas || newReplicaCount != oldReplicaCount || lastReplicas != newReplicaCount {
			copied, err := deploymentCopy(&deployment)
			if err != nil {
				glog.V(2).Infof("Deep copy of deployment %q failed: %v", deployment.Name, err)
				return err
			}

			copied.Spec.Replicas = newReplicaCount
			copied.Annotations[deployapi.DeploymentReplicasAnnotation] = strconv.Itoa(int(newReplicaCount))

			if _, err := c.rn.ReplicationControllers(copied.Namespace).Update(copied); err != nil {
				c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentScaleFailed",
					"Failed to scale deployment %q from %d to %d: %v", copied.Name, oldReplicaCount, newReplicaCount, err)
				return err
			}
			// Only report scaling events if we changed the replica count.
			if oldReplicaCount != newReplicaCount {
				c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentScaled",
					"Scaled deployment %q from %d to %d", copied.Name, oldReplicaCount, newReplicaCount)
			} else {
				glog.V(4).Infof("Updated deployment %q replica annotation to match current replica count %d", deployutil.LabelForDeployment(copied), newReplicaCount)
			}
		}
	}

	return c.updateStatus(config)
}

func (c *DeploymentConfigController) updateStatus(config *deployapi.DeploymentConfig) error {
	// NOTE: We should update the status of the deployment config only if we need to, otherwise
	// we hotloop between updates.
	if !needsUpdate(config) {
		return nil
	}
	config.Status.ObservedGeneration = config.Generation
	if _, err := c.dn.DeploymentConfigs(config.Namespace).UpdateStatus(config); err != nil {
		glog.V(2).Infof("Cannot update the status for %q: %v", deployutil.LabelForDeploymentConfig(config), err)
		return err
	}
	glog.V(4).Infof("Updated the status for %q (observed generation: %d)", deployutil.LabelForDeploymentConfig(config), config.Status.ObservedGeneration)
	return nil
}

func (c *DeploymentConfigController) handleErr(err error, key interface{}) {
	if err == nil {
		return
	}
	if _, isFatal := err.(fatalError); isFatal {
		utilruntime.HandleError(err)
		c.queue.Forget(key)
		return
	}

	if c.queue.NumRequeues(key) < 10 {
		c.queue.AddRateLimited(key)
	} else {
		glog.V(2).Infof(err.Error())
		c.queue.Forget(key)
	}
}

func needsUpdate(config *deployapi.DeploymentConfig) bool {
	return config.Generation > config.Status.ObservedGeneration
}

func deploymentCopy(rc *kapi.ReplicationController) (*kapi.ReplicationController, error) {
	objCopy, err := kapi.Scheme.DeepCopy(rc)
	if err != nil {
		return nil, err
	}
	copied, ok := objCopy.(*kapi.ReplicationController)
	if !ok {
		return nil, fmt.Errorf("expected ReplicationController, got %#v", objCopy)
	}
	return copied, nil
}
<commit_msg>dc controller was mutating cache objects<commit_after>package deploymentconfig

import (
	"fmt"
	"strconv"

	"github.com/golang/glog"

	kapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/client/record"
	kclient "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/runtime"
	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
	"k8s.io/kubernetes/pkg/util/workqueue"

	osclient "github.com/openshift/origin/pkg/client"
	oscache "github.com/openshift/origin/pkg/client/cache"
	deployapi "github.com/openshift/origin/pkg/deploy/api"
	deployutil "github.com/openshift/origin/pkg/deploy/util"
)

// fatalError is an error which can't be retried.
type fatalError string

func (e fatalError) Error() string {
	return fmt.Sprintf("fatal error handling deployment config: %s", string(e))
}

// DeploymentConfigController is responsible for creating a new deployment
// when:
//
//    1. The config version is > 0 and,
//    2. No deployment for the version exists.
//
// The controller reconciles deployments with the replica count specified on
// the config. The active deployment (that is, the latest successful
// deployment) will always be scaled to the config replica count. All other
// deployments will be scaled to zero.
//
// If a new version is observed for which no deployment exists, any running
// deployments will be cancelled. The controller will not attempt to scale
// running deployments.
type DeploymentConfigController struct {
	// dn provides access to deploymentconfigs.
	dn osclient.DeploymentConfigsNamespacer
	// rn provides access to replication controllers.
	rn kclient.ReplicationControllersNamespacer

	// queue contains deployment configs that need to be synced.
	queue workqueue.RateLimitingInterface

	// dcStore provides a local cache for deployment configs.
	dcStore oscache.StoreToDeploymentConfigLister
	// rcStore provides a local cache for replication controllers.
	rcStore cache.StoreToReplicationControllerLister
	// dcStoreSynced makes sure the dc store is synced before reconcling any deployment config.
	dcStoreSynced func() bool
	// rcStoreSynced makes sure the rc store is synced before reconcling any deployment config.
	rcStoreSynced func() bool

	// codec is used to build deployments from configs.
	codec runtime.Codec
	// recorder is used to record events.
	recorder record.EventRecorder
}

// Handle implements the loop that processes deployment configs. Since this controller started
// using caches, the provided config MUST be deep-copied beforehand (see work() in factory.go).
func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error {
	// There's nothing to reconcile until the version is nonzero or when the
	// deployment config has been marked for deletion.
	if config.Status.LatestVersion == 0 || config.DeletionTimestamp != nil {
		return c.updateStatus(config)
	}

	// Find all deployments owned by the deployment config.
	selector := deployutil.ConfigSelector(config.Name)
	existingDeployments, err := c.rcStore.ReplicationControllers(config.Namespace).List(selector)
	if err != nil {
		return err
	}

	latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments)
	// If the latest deployment doesn't exist yet, cancel any running
	// deployments to allow them to be superceded by the new config version.
	awaitingCancellations := false
	if !latestIsDeployed {
		for i := range existingDeployments {
			deployment := existingDeployments[i]
			// Skip deployments with an outcome.
			if deployutil.IsTerminatedDeployment(&deployment) {
				continue
			}
			// Cancel running deployments.
			awaitingCancellations = true
			if !deployutil.IsDeploymentCancelled(&deployment) {
				copied, err := deploymentCopy(&deployment)
				if err != nil {
					return err
				}

				copied.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue
				copied.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists

				updatedDeployment, err := c.rn.ReplicationControllers(copied.Namespace).Update(copied)
				if err != nil {
					c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCancellationFailed", "Failed to cancel deployment %q superceded by version %d: %s", deployment.Name, config.Status.LatestVersion, err)
				} else {
					// replace the current deployment with the updated copy so that a future update has a chance at working
					existingDeployments[i] = *updatedDeployment
					c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCancelled", "Cancelled deployment %q superceded by version %d", deployment.Name, config.Status.LatestVersion)
				}
			}
		}
	}
	// Wait for deployment cancellations before reconciling or creating a new
	// deployment to avoid competing with existing deployment processes.
	if awaitingCancellations {
		c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentAwaitingCancellation", "Deployment of version %d awaiting cancellation of older running deployments", config.Status.LatestVersion)
		return fmt.Errorf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config))
	}
	// If the latest deployment already exists, reconcile existing deployments
	// and return early.
	if latestIsDeployed {
		// If the latest deployment is still running, try again later. We don't
		// want to compete with the deployer.
		if !deployutil.IsTerminatedDeployment(latestDeployment) {
			return c.updateStatus(config)
		}
		return c.reconcileDeployments(existingDeployments, config)
	}
	// If the config is paused we shouldn't create new deployments for it.
	// TODO: Make sure cleanup policy will work for paused configs.
	if config.Spec.Paused {
		return c.updateStatus(config)
	}
	// No deployments are running and the latest deployment doesn't exist, so
	// create the new deployment.
	deployment, err := deployutil.MakeDeployment(config, c.codec)
	if err != nil {
		return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err))
	}
	created, err := c.rn.ReplicationControllers(config.Namespace).Create(deployment)
	if err != nil {
		// If the deployment was already created, just move on. The cache could be
		// stale, or another process could have already handled this update.
		if errors.IsAlreadyExists(err) {
			return c.updateStatus(config)
		}
		c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCreationFailed", "Couldn't deploy version %d: %s", config.Status.LatestVersion, err)
		return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)
	}
	c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCreated", "Created new deployment %q for version %d", created.Name, config.Status.LatestVersion)

	return c.updateStatus(config)
}

// reconcileDeployments reconciles existing deployment replica counts which
// could have diverged outside the deployment process (e.g. due to auto or
// manual scaling, or partial deployments). The active deployment is the last
// successful deployment, not necessarily the latest in terms of the config
// version. The active deployment replica count should follow the config, and
// all other deployments should be scaled to zero.
//
// Previously, scaling behavior was that the config replica count was used
// only for initial deployments and the active deployment had to be scaled up
// directly. To continue supporting that old behavior we must detect when the
// deployment has been directly manipulated, and if so, preserve the directly
// updated value and sync the config with the deployment.
func (c *DeploymentConfigController) reconcileDeployments(existingDeployments []kapi.ReplicationController, config *deployapi.DeploymentConfig) error {
	latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments)
	if !latestIsDeployed {
		// We shouldn't be reconciling if the latest deployment hasn't been
		// created; this is enforced on the calling side, but double checking
		// can't hurt.
		return nil
	}
	activeDeployment := deployutil.ActiveDeployment(config, existingDeployments)
	// Compute the replica count for the active deployment (even if the active
	// deployment doesn't exist). The active replica count is the value that
	// should be assigned to the config, to allow the replica propagation to
	// flow downward from the config.
	//
	// By default we'll assume the config replicas should be used to update the
	// active deployment except in special cases (like first sync or externally
	// updated deployments.)
	activeReplicas := config.Spec.Replicas
	source := "the deploymentConfig itself (no change)"

	activeDeploymentExists := activeDeployment != nil
	activeDeploymentIsLatest := activeDeploymentExists && activeDeployment.Name == latestDeployment.Name
	latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment)

	switch {
	case activeDeploymentExists && activeDeploymentIsLatest:
		// The active/latest deployment follows the config unless this is its first
		// sync or if an external change to the deployment replicas is detected.
		lastActiveReplicas, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment)
		if !hasLastActiveReplicas || lastActiveReplicas != activeDeployment.Spec.Replicas {
			activeReplicas = activeDeployment.Spec.Replicas
			source = fmt.Sprintf("the latest/active deployment %q which was scaled directly or has not previously been synced", deployutil.LabelForDeployment(activeDeployment))
		}
	case activeDeploymentExists && !activeDeploymentIsLatest:
		// The active/non-latest deployment follows the config if it was
		// previously synced; if this is the first sync, infer what the config
		// value should be based on either the latest desired or whatever the
		// deployment is currently scaled to.
		_, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment)
		if hasLastActiveReplicas {
			break
		}
		if latestHasDesiredReplicas {
			activeReplicas = latestDesiredReplicas
			source = fmt.Sprintf("the desired replicas of latest deployment %q which has not been previously synced", deployutil.LabelForDeployment(latestDeployment))
		} else if activeDeployment.Spec.Replicas > 0 {
			activeReplicas = activeDeployment.Spec.Replicas
			source = fmt.Sprintf("the active deployment %q which has not been previously synced", deployutil.LabelForDeployment(activeDeployment))
		}
	case !activeDeploymentExists && latestHasDesiredReplicas:
		// If there's no active deployment, use the latest desired, if available.
		activeReplicas = latestDesiredReplicas
		source = fmt.Sprintf("the desired replicas of latest deployment %q with no active deployment", deployutil.LabelForDeployment(latestDeployment))
	}

	// Bring the config in sync with the deployment. Once we know the config
	// accurately represents the desired replica count of the active deployment,
	// we can safely reconcile deployments.
	//
	// If the deployment config is test, never update the deployment config based
	// on deployments, since test behavior overrides user scaling.
	switch {
	case config.Spec.Replicas == activeReplicas:
	case config.Spec.Test:
		glog.V(4).Infof("Detected changed replicas for test deploymentConfig %q, ignoring that change", deployutil.LabelForDeploymentConfig(config))
	default:
		oldReplicas := config.Spec.Replicas
		config.Spec.Replicas = activeReplicas
		var err error
		config, err = c.dn.DeploymentConfigs(config.Namespace).Update(config)
		if err != nil {
			return err
		}
		glog.V(4).Infof("Synced deploymentConfig %q replicas from %d to %d based on %s", deployutil.LabelForDeploymentConfig(config), oldReplicas, activeReplicas, source)
	}

	// Reconcile deployments. The active deployment follows the config, and all
	// other deployments should be scaled to zero.
	for _, deployment := range existingDeployments {
		isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name

		oldReplicaCount := deployment.Spec.Replicas
		newReplicaCount := int32(0)
		if isActiveDeployment {
			newReplicaCount = activeReplicas
		}
		if config.Spec.Test {
			glog.V(4).Infof("Deployment config %q is test and deployment %q will be scaled down", deployutil.LabelForDeploymentConfig(config), deployutil.LabelForDeployment(&deployment))
			newReplicaCount = 0
		}
		lastReplicas, hasLastReplicas := deployutil.DeploymentReplicas(&deployment)
		// Only update if necessary.
		if !hasLastReplicas || newReplicaCount != oldReplicaCount || lastReplicas != newReplicaCount {
			copied, err := deploymentCopy(&deployment)
			if err != nil {
				glog.V(2).Infof("Deep copy of deployment %q failed: %v", deployment.Name, err)
				return err
			}

			copied.Spec.Replicas = newReplicaCount
			copied.Annotations[deployapi.DeploymentReplicasAnnotation] = strconv.Itoa(int(newReplicaCount))

			if _, err := c.rn.ReplicationControllers(copied.Namespace).Update(copied); err != nil {
				c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentScaleFailed",
					"Failed to scale deployment %q from %d to %d: %v", copied.Name, oldReplicaCount, newReplicaCount, err)
				return err
			}
			// Only report scaling events if we changed the replica count.
			if oldReplicaCount != newReplicaCount {
				c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentScaled",
					"Scaled deployment %q from %d to %d", copied.Name, oldReplicaCount, newReplicaCount)
			} else {
				glog.V(4).Infof("Updated deployment %q replica annotation to match current replica count %d", deployutil.LabelForDeployment(copied), newReplicaCount)
			}
		}
	}

	return c.updateStatus(config)
}

func (c *DeploymentConfigController) updateStatus(config *deployapi.DeploymentConfig) error {
	// NOTE: We should update the status of the deployment config only if we need to, otherwise
	// we hotloop between updates.
	if !needsUpdate(config) {
		return nil
	}
	config.Status.ObservedGeneration = config.Generation
	if _, err := c.dn.DeploymentConfigs(config.Namespace).UpdateStatus(config); err != nil {
		glog.V(2).Infof("Cannot update the status for %q: %v", deployutil.LabelForDeploymentConfig(config), err)
		return err
	}
	glog.V(4).Infof("Updated the status for %q (observed generation: %d)", deployutil.LabelForDeploymentConfig(config), config.Status.ObservedGeneration)
	return nil
}

func (c *DeploymentConfigController) handleErr(err error, key interface{}) {
	if err == nil {
		return
	}
	if _, isFatal := err.(fatalError); isFatal {
		utilruntime.HandleError(err)
		c.queue.Forget(key)
		return
	}

	if c.queue.NumRequeues(key) < 10 {
		c.queue.AddRateLimited(key)
	} else {
		glog.V(2).Infof(err.Error())
		c.queue.Forget(key)
	}
}

func needsUpdate(config *deployapi.DeploymentConfig) bool {
	return config.Generation > config.Status.ObservedGeneration
}

func deploymentCopy(rc *kapi.ReplicationController) (*kapi.ReplicationController, error) {
	objCopy, err := kapi.Scheme.DeepCopy(rc)
	if err != nil {
		return nil, err
	}
	copied, ok := objCopy.(*kapi.ReplicationController)
	if !ok {
		return nil, fmt.Errorf("expected ReplicationController, got %#v", objCopy)
	}
	return copied, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"github.com/mattbaird/elastigo/api"
	"github.com/mattbaird/elastigo/core"
	"log"
	"os"
)

var (
	host *string = flag.String("host", "localhost", "Elasticsearch Host")
)

func main() {
	core.DebugRequests = true
	log.SetFlags(log.LstdFlags)
	flag.Parse()

	fmt.Println("host = ", *host)
	// Set the Elasticsearch Host to Connect to
	api.Domain = *host

	// Index a document
	_, err := core.Index("testindex", "user", "docid_1", nil, `{"name":"bob"}`)
	exitIfErr(err)

	// Index a doc using a map of values
	_, err = core.Index("testindex", "user", "docid_2", nil, map[string]string{"name": "venkatesh"})
	exitIfErr(err)

	// Index a doc using Structs
	_, err = core.Index("testindex", "user", "docid_3", nil, MyUser{"wanda", 22})
	exitIfErr(err)

	// Search Using Raw json String
	searchJson := `{
	    "query" : {
	        "term" : { "Name" : "wanda" }
	    }
	}`
	out, err := core.SearchRequest("testindex", "user", nil, searchJson)
	if len(out.Hits.Hits) == 1 {
		fmt.Println(string(out.Hits.Hits[0].Source))
	}
	exitIfErr(err)

}
func exitIfErr(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s\n", err.Error())
		os.Exit(1)
	}
}

type MyUser struct {
	Name string
	Age  int
}
<commit_msg>use %v to print out value instead of casting to string<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"github.com/mattbaird/elastigo/api"
	"github.com/mattbaird/elastigo/core"
	"log"
	"os"
)

var (
	host *string = flag.String("host", "localhost", "Elasticsearch Host")
)

func main() {
	core.DebugRequests = true
	log.SetFlags(log.LstdFlags)
	flag.Parse()

	fmt.Println("host = ", *host)
	// Set the Elasticsearch Host to Connect to
	api.Domain = *host

	// Index a document
	_, err := core.Index("testindex", "user", "docid_1", nil, `{"name":"bob"}`)
	exitIfErr(err)

	// Index a doc using a map of values
	_, err = core.Index("testindex", "user", "docid_2", nil, map[string]string{"name": "venkatesh"})
	exitIfErr(err)

	// Index a doc using Structs
	_, err = core.Index("testindex", "user", "docid_3", nil, MyUser{"wanda", 22})
	exitIfErr(err)

	// Search Using Raw json String
	searchJson := `{
	    "query" : {
	        "term" : { "Name" : "wanda" }
	    }
	}`
	out, err := core.SearchRequest("testindex", "user", nil, searchJson)
	if len(out.Hits.Hits) == 1 {
		fmt.Println("%v", out.Hits.Hits[0].Source)
	}
	exitIfErr(err)

}
func exitIfErr(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s\n", err.Error())
		os.Exit(1)
	}
}

type MyUser struct {
	Name string
	Age  int
}
<|endoftext|>


<commit_before>// combat utility functions

package main

func (g *game) HitDamage(base int, armor int) int {
	min := base / 2
	attack := min + RandInt(base-min+1)
	attack -= RandInt(armor + 1)
	if attack < 0 {
		attack = 0
	}
	return attack
}

func (m *monster) InflictDamage(g *game, damage, max int) {
	oldHP := g.Player.HP
	g.Player.HP -= damage
	if oldHP > max && g.Player.HP <= max {
		g.StoryPrintf("Critical HP: %d (hit by %s)", g.Player.HP, Indefinite(m.Kind.String(), false))
		g.ui.CriticalHPWarning(g)
	}
}

func (g *game) MakeMonstersAware() {
	for _, m := range g.Monsters {
		if m.HP <= 0 {
			continue
		}
		if g.Player.LOS[m.Pos] {
			m.MakeAware(g)
			if m.State != Resting {
				m.GatherBand(g)
			}
		}
	}
}

func (g *game) MakeNoise(noise int, at position) {
	dij := &normalPath{game: g}
	nm := Dijkstra(dij, []position{at}, noise)
	for _, m := range g.Monsters {
		if !m.Exists() {
			continue
		}
		if m.State == Hunting {
			continue
		}
		n, ok := nm[m.Pos]
		if !ok {
			continue
		}
		d := n.Cost
		v := noise - d
		if v <= 0 {
			continue
		}
		v *= 3
		if v > 90 {
			v = 90
		}
		r := RandInt(100)
		if m.State == Resting {
			r += 10
		}
		if v > r {
			m.Target = at
			if g.Player.LOS[m.Pos] {
				m.State = Hunting
			} else {
				m.State = Wandering
			}
			m.GatherBand(g)
		}
	}
}

func (g *game) AttackMonster(mons *monster) {
	switch {
	case g.Player.Weapon.Cleave():
		var neighbors []position
		if g.Player.HasStatus(StatusConfusion) {
			neighbors = g.Dungeon.CardinalFreeNeighbors(g.Player.Pos)
		} else {
			neighbors = g.Dungeon.FreeNeighbors(g.Player.Pos)
		}
		for _, pos := range neighbors {
			mons, _ := g.MonsterAt(pos)
			if mons.Exists() {
				g.HitMonster(mons)
			}
		}
	case g.Player.Weapon.Pierce():
		g.HitMonster(mons)
		deltaX := mons.Pos.X - g.Player.Pos.X
		deltaY := mons.Pos.Y - g.Player.Pos.Y
		behind := position{g.Player.Pos.X + 2*deltaX, g.Player.Pos.Y + 2*deltaY}
		if g.Dungeon.Valid(behind) {
			mons, _ := g.MonsterAt(behind)
			if mons.Exists() {
				g.HitMonster(mons)
			}
		}
	default:
		g.HitMonster(mons)
		if (g.Player.Weapon == Sword || g.Player.Weapon == DoubleSword) && RandInt(4) == 0 {
			g.HitMonster(mons)
		}
	}
}

func (g *game) HitMonster(mons *monster) {
	acc := RandInt(g.Player.Accuracy())
	ev := RandInt(mons.Evasion)
	if mons.State == Resting {
		ev /= 2 + 1
	}
	if acc > ev {
		g.MakeNoise(12, mons.Pos)
		bonus := 0
		if g.Player.HasStatus(StatusBerserk) {
			bonus += RandInt(5)
		}
		attack := g.HitDamage(g.Player.Attack()+bonus, mons.Armor)
		if mons.State == Resting {
			if g.Player.Weapon == Dagger {
				attack *= 4
			} else {
				attack *= 2
			}
		}
		oldHP := mons.HP
		mons.HP -= attack
		if mons.HP > 0 {
			g.Printf("You hit the %v (%d damage).", mons.Kind, attack)
		} else if oldHP > 0 {
			// test oldHP > 0 because of sword special attack
			g.Printf("You kill the %v (%d damage).", mons.Kind, attack)
			g.HandleKill(mons)
		}
	} else {
		g.Printf("You miss the %v.", mons.Kind)
	}
	mons.MakeHuntIfHurt(g)
}

func (g *game) HandleKill(mons *monster) {
	g.Killed++
	if g.KilledMons == nil {
		g.KilledMons = map[monsterKind]int{}
	}
	g.KilledMons[mons.Kind]++
	if mons.Kind == MonsExplosiveNadre {
		mons.Explode(g)
	}
	if mons.Kind.Dangerousness() > 10 {
		g.StoryPrintf("You killed %s.", Indefinite(mons.Kind.String(), false))
	}
}
<commit_msg>improve berserk bonus damage a little<commit_after>// combat utility functions

package main

func (g *game) HitDamage(base int, armor int) int {
	min := base / 2
	attack := min + RandInt(base-min+1)
	attack -= RandInt(armor + 1)
	if attack < 0 {
		attack = 0
	}
	return attack
}

func (m *monster) InflictDamage(g *game, damage, max int) {
	oldHP := g.Player.HP
	g.Player.HP -= damage
	if oldHP > max && g.Player.HP <= max {
		g.StoryPrintf("Critical HP: %d (hit by %s)", g.Player.HP, Indefinite(m.Kind.String(), false))
		g.ui.CriticalHPWarning(g)
	}
}

func (g *game) MakeMonstersAware() {
	for _, m := range g.Monsters {
		if m.HP <= 0 {
			continue
		}
		if g.Player.LOS[m.Pos] {
			m.MakeAware(g)
			if m.State != Resting {
				m.GatherBand(g)
			}
		}
	}
}

func (g *game) MakeNoise(noise int, at position) {
	dij := &normalPath{game: g}
	nm := Dijkstra(dij, []position{at}, noise)
	for _, m := range g.Monsters {
		if !m.Exists() {
			continue
		}
		if m.State == Hunting {
			continue
		}
		n, ok := nm[m.Pos]
		if !ok {
			continue
		}
		d := n.Cost
		v := noise - d
		if v <= 0 {
			continue
		}
		v *= 3
		if v > 90 {
			v = 90
		}
		r := RandInt(100)
		if m.State == Resting {
			r += 10
		}
		if v > r {
			m.Target = at
			if g.Player.LOS[m.Pos] {
				m.State = Hunting
			} else {
				m.State = Wandering
			}
			m.GatherBand(g)
		}
	}
}

func (g *game) AttackMonster(mons *monster) {
	switch {
	case g.Player.Weapon.Cleave():
		var neighbors []position
		if g.Player.HasStatus(StatusConfusion) {
			neighbors = g.Dungeon.CardinalFreeNeighbors(g.Player.Pos)
		} else {
			neighbors = g.Dungeon.FreeNeighbors(g.Player.Pos)
		}
		for _, pos := range neighbors {
			mons, _ := g.MonsterAt(pos)
			if mons.Exists() {
				g.HitMonster(mons)
			}
		}
	case g.Player.Weapon.Pierce():
		g.HitMonster(mons)
		deltaX := mons.Pos.X - g.Player.Pos.X
		deltaY := mons.Pos.Y - g.Player.Pos.Y
		behind := position{g.Player.Pos.X + 2*deltaX, g.Player.Pos.Y + 2*deltaY}
		if g.Dungeon.Valid(behind) {
			mons, _ := g.MonsterAt(behind)
			if mons.Exists() {
				g.HitMonster(mons)
			}
		}
	default:
		g.HitMonster(mons)
		if (g.Player.Weapon == Sword || g.Player.Weapon == DoubleSword) && RandInt(4) == 0 {
			g.HitMonster(mons)
		}
	}
}

func (g *game) HitMonster(mons *monster) {
	acc := RandInt(g.Player.Accuracy())
	ev := RandInt(mons.Evasion)
	if mons.State == Resting {
		ev /= 2 + 1
	}
	if acc > ev {
		g.MakeNoise(12, mons.Pos)
		bonus := 0
		if g.Player.HasStatus(StatusBerserk) {
			bonus += 2 + RandInt(4)
		}
		attack := g.HitDamage(g.Player.Attack()+bonus, mons.Armor)
		if mons.State == Resting {
			if g.Player.Weapon == Dagger {
				attack *= 4
			} else {
				attack *= 2
			}
		}
		oldHP := mons.HP
		mons.HP -= attack
		if mons.HP > 0 {
			g.Printf("You hit the %v (%d damage).", mons.Kind, attack)
		} else if oldHP > 0 {
			// test oldHP > 0 because of sword special attack
			g.Printf("You kill the %v (%d damage).", mons.Kind, attack)
			g.HandleKill(mons)
		}
	} else {
		g.Printf("You miss the %v.", mons.Kind)
	}
	mons.MakeHuntIfHurt(g)
}

func (g *game) HandleKill(mons *monster) {
	g.Killed++
	if g.KilledMons == nil {
		g.KilledMons = map[monsterKind]int{}
	}
	g.KilledMons[mons.Kind]++
	if mons.Kind == MonsExplosiveNadre {
		mons.Explode(g)
	}
	if mons.Kind.Dangerousness() > 10 {
		g.StoryPrintf("You killed %s.", Indefinite(mons.Kind.String(), false))
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

const (
	whiteKingSafety    = 0x01  // Should we worry about white king's safety?
	blackKingSafety    = 0x02  // Ditto for the black king.
	materialDraw       = 0x04  // King vs. King (with minor)
	knownEndgame       = 0x08  // Where we calculate exact score.
	lesserKnownEndgame = 0x10  // Where we set score markdown value.
	singleBishops      = 0x20  // Sides might have bishops on opposite color squares.
)

// Hash containing various evaluation metrics; used only when evaluation tracing
// is enabled.
type Metrics map[string]interface{}

// King safety information; used only in the middle game when there is enough
// material to worry about the king safety.
type Safety struct {
	fort Bitmask 		// Squares around the king plus one extra row in front.
	threats int 		// A sum of treats: each based on attacking piece type.
	attacks int 		// Number of attacks on squares adjacent to the king.
	attackers int 		// Number of pieces attacking king's fort.
}

// Helper structure used for evaluation tracking.
type Total struct {
	white Score 		// Score for white.
	black Score 		// Score for black.
}

type Function func(*Evaluation) int
type MaterialEntry struct {
	score     Score 	// Score adjustment for the given material.
	endgame   Function 	// Function to analyze an endgame position.
	phase     int 		// Game phase, based on available material.
	flags     uint8    	// Evaluation flags based on material balance.
}

type Evaluation struct {
	score     Score 	 // Current score.
	safety    [2]Safety 	 // King safety data for both sides.
	attacks   [14]Bitmask 	 // Attack bitmasks for all the pieces on the board.
	pawns     *PawnEntry 	 // Pointer to the pawn cache entry.
	material  *MaterialEntry // Pointer to the matrial base entry.
	position  *Position 	 // Pointer to the position we're evaluating.
	metrics   Metrics 	 // Evaluation metrics when tracking is on.
}

// Use single statically allocated variable to avoid garbage collection overhead.
var eval Evaluation

// Main position evaluation method that returns single blended score.
func (p *Position) Evaluate() int {
	return eval.init(p).run()
}

// Auxiliary evaluation method that captures individual evaluation metrics. This
// is useful when we want to see evaluation summary.
func (p *Position) EvaluateWithTrace() (int, Metrics) {
	eval.init(p)
	eval.metrics = make(Metrics)

	engine.trace = true
	defer func() {
		var tempo Total
		var final Score

		if p.color == White {
			tempo.white.add(rightToMove)
			final.add(eval.score)
		} else {
			tempo.black.add(rightToMove)
			final.subtract(eval.score)
		}

		eval.checkpoint(`Phase`, eval.material.phase)
		eval.checkpoint(`PST`, p.tally)
		eval.checkpoint(`Tempo`, tempo)
		eval.checkpoint(`Final`, final)
		engine.trace = false
	}()

	return eval.run(), eval.metrics
}

func (e *Evaluation) init(p *Position) *Evaluation {
	eval = Evaluation{}
	e.position = p

	// Initialize the score with incremental PST value and right to move.
	e.score = p.tally
	if e.position.color == White {
		e.score.add(rightToMove)
	} else {
		e.score.subtract(rightToMove)
	}

	// Set up king and pawn attacks for both sides.
	e.attacks[King] = p.kingAttacks(White)
	e.attacks[Pawn] = p.pawnAttacks(White)
	e.attacks[BlackKing] = p.kingAttacks(Black)
	e.attacks[BlackPawn] = p.pawnAttacks(Black)

	// Overall attacks for both sides include kings and pawns so far.
	e.attacks[White] = e.attacks[King] | e.attacks[Pawn]
	e.attacks[Black] = e.attacks[BlackKing] | e.attacks[BlackPawn]

	return e
}

func (e *Evaluation) analyzeMaterialNew() {
	e.material = &materialBase[e.position.balance]
	e.score.add(e.material.score)
}

func (e *Evaluation) run() int {
	e.material = &materialBase[e.position.balance]
	if e.material.flags & materialDraw != 0 {
		return 0
	}

	e.score.add(e.material.score)
	if e.material.flags & knownEndgame != 0 {
		return e.evaluateEndgame()
	}

	e.analyzePawns()
	e.analyzePieces()
	e.analyzeThreats()
	e.analyzeSafety()
	e.analyzePassers()
	e.wrapUp()

	return e.score.blended(e.material.phase)
}

func (e *Evaluation) wrapUp() {

	// Adjust the score if we have lesser known endgame.
	if e.material.flags & lesserKnownEndgame != 0 {
		e.inspectEndgame()
	}

	// Flip the sign for black so that blended evaluation score always
	// represents the white side.
	if e.position.color == Black {
		e.score.midgame = -e.score.midgame
		e.score.endgame = -e.score.endgame
	}
}

func (e *Evaluation) checkpoint(tag string, metric interface{}) {
	e.metrics[tag] = metric
}

func (e *Evaluation) oppositeBishops() bool {
	bishops := e.position.outposts[Bishop] | e.position.outposts[BlackBishop]

	return bishops & maskDark == 0 || bishops & ^maskDark == 0
}
<commit_msg>Evaluation tweak to check for insufficiant material first<commit_after>// Copyright (c) 2013-2014 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

const (
	whiteKingSafety    = 0x01  // Should we worry about white king's safety?
	blackKingSafety    = 0x02  // Ditto for the black king.
	materialDraw       = 0x04  // King vs. King (with minor)
	knownEndgame       = 0x08  // Where we calculate exact score.
	lesserKnownEndgame = 0x10  // Where we set score markdown value.
	singleBishops      = 0x20  // Sides might have bishops on opposite color squares.
)

// Hash containing various evaluation metrics; used only when evaluation tracing
// is enabled.
type Metrics map[string]interface{}

// King safety information; used only in the middle game when there is enough
// material to worry about the king safety.
type Safety struct {
	fort Bitmask 		// Squares around the king plus one extra row in front.
	threats int 		// A sum of treats: each based on attacking piece type.
	attacks int 		// Number of attacks on squares adjacent to the king.
	attackers int 		// Number of pieces attacking king's fort.
}

// Helper structure used for evaluation tracking.
type Total struct {
	white Score 		// Score for white.
	black Score 		// Score for black.
}

type Function func(*Evaluation) int
type MaterialEntry struct {
	score     Score 	// Score adjustment for the given material.
	endgame   Function 	// Function to analyze an endgame position.
	phase     int 		// Game phase, based on available material.
	flags     uint8    	// Evaluation flags based on material balance.
}

type Evaluation struct {
	score     Score 	 // Current score.
	safety    [2]Safety 	 // King safety data for both sides.
	attacks   [14]Bitmask 	 // Attack bitmasks for all the pieces on the board.
	pawns     *PawnEntry 	 // Pointer to the pawn cache entry.
	material  *MaterialEntry // Pointer to the matrial base entry.
	position  *Position 	 // Pointer to the position we're evaluating.
	metrics   Metrics 	 // Evaluation metrics when tracking is on.
}

// Use single statically allocated variable to avoid garbage collection overhead.
var eval Evaluation

// Main position evaluation method that returns single blended score.
func (p *Position) Evaluate() int {
	if p.insufficient() {
		return 0
	}
	return eval.init(p).run()
}

// Auxiliary evaluation method that captures individual evaluation metrics. This
// is useful when we want to see evaluation summary.
func (p *Position) EvaluateWithTrace() (int, Metrics) {
	eval.init(p)
	eval.metrics = make(Metrics)

	engine.trace = true
	defer func() {
		var tempo Total
		var final Score

		if p.color == White {
			tempo.white.add(rightToMove)
			final.add(eval.score)
		} else {
			tempo.black.add(rightToMove)
			final.subtract(eval.score)
		}

		eval.checkpoint(`Phase`, eval.material.phase)
		eval.checkpoint(`PST`, p.tally)
		eval.checkpoint(`Tempo`, tempo)
		eval.checkpoint(`Final`, final)
		engine.trace = false
	}()

	return eval.run(), eval.metrics
}

func (e *Evaluation) init(p *Position) *Evaluation {
	eval = Evaluation{}
	e.position = p

	// Initialize the score with incremental PST value and right to move.
	e.score = p.tally
	if e.position.color == White {
		e.score.add(rightToMove)
	} else {
		e.score.subtract(rightToMove)
	}

	// Set up king and pawn attacks for both sides.
	e.attacks[King] = p.kingAttacks(White)
	e.attacks[Pawn] = p.pawnAttacks(White)
	e.attacks[BlackKing] = p.kingAttacks(Black)
	e.attacks[BlackPawn] = p.pawnAttacks(Black)

	// Overall attacks for both sides include kings and pawns so far.
	e.attacks[White] = e.attacks[King] | e.attacks[Pawn]
	e.attacks[Black] = e.attacks[BlackKing] | e.attacks[BlackPawn]

	return e
}

func (e *Evaluation) run() int {
	e.material = &materialBase[e.position.balance]

	e.score.add(e.material.score)
	if e.material.flags & knownEndgame != 0 {
		return e.evaluateEndgame()
	}

	e.analyzePawns()
	e.analyzePieces()
	e.analyzeThreats()
	e.analyzeSafety()
	e.analyzePassers()
	e.wrapUp()

	return e.score.blended(e.material.phase)
}

func (e *Evaluation) wrapUp() {

	// Adjust the score if we have lesser known endgame.
	if e.material.flags & lesserKnownEndgame != 0 {
		e.inspectEndgame()
	}

	// Flip the sign for black so that blended evaluation score always
	// represents the white side.
	if e.position.color == Black {
		e.score.midgame = -e.score.midgame
		e.score.endgame = -e.score.endgame
	}
}

func (e *Evaluation) checkpoint(tag string, metric interface{}) {
	e.metrics[tag] = metric
}

func (e *Evaluation) oppositeBishops() bool {
	bishops := e.position.outposts[Bishop] | e.position.outposts[BlackBishop]

	return bishops & maskDark == 0 || bishops & ^maskDark == 0
}
<|endoftext|>


<commit_before>package handlers

import (
	"context"
	"io"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/openfaas/faas/gateway/metrics"
	"github.com/openfaas/faas/gateway/types"
	"github.com/prometheus/client_golang/prometheus"
)

// MakeForwardingProxyHandler create a handler which forwards HTTP requests
func MakeForwardingProxyHandler(proxy *types.HTTPClientReverseProxy, metrics *metrics.MetricOptions) http.HandlerFunc {
	baseURL := proxy.BaseURL.String()
	if strings.HasSuffix(baseURL, "/") {
		baseURL = baseURL[0 : len(baseURL)-1]
	}

	return func(w http.ResponseWriter, r *http.Request) {

		requestURL := r.URL.String()
		serviceName := getServiceName(requestURL)

		log.Printf("> Forwarding [%s] to %s", r.Method, requestURL)

		start := time.Now()

		statusCode, err := forwardRequest(w, r, proxy.Client, baseURL, requestURL, proxy.Timeout)

		if err != nil {
			log.Printf("error with upstream request to: %s, %s\n", requestURL, err.Error())
		}

		seconds := time.Since(start).Seconds()
		log.Printf("< [%s] - %d took %f seconds\n", r.URL.String(),
			statusCode, seconds)

		if len(serviceName) > 0 {
			metrics.GatewayFunctionsHistogram.
				WithLabelValues(serviceName).
				Observe(seconds)

			code := strconv.Itoa(statusCode)

			metrics.GatewayFunctionInvocation.
				With(prometheus.Labels{"function_name": serviceName, "code": code}).
				Inc()
		}

	}
}

func forwardRequest(w http.ResponseWriter, r *http.Request, proxyClient *http.Client, baseURL string, requestURL string, timeout time.Duration) (int, error) {

	upstreamReq, _ := http.NewRequest(r.Method, baseURL+requestURL, nil)

	upstreamReq.Header["X-Forwarded-For"] = []string{r.RequestURI}

	if r.Body != nil {
		defer r.Body.Close()
		upstreamReq.Body = r.Body
	}

	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	res, resErr := proxyClient.Do(upstreamReq.WithContext(ctx))
	if resErr != nil {
		badStatus := http.StatusBadGateway
		w.WriteHeader(badStatus)
		return badStatus, resErr
	}

	if res.Body != nil {
		defer res.Body.Close()
	}

	// Populate any headers received
	for k, v := range res.Header {
		w.Header()[k] = v
	}

	// Write status code
	w.WriteHeader(res.StatusCode)

	if res.Body != nil {
		// Copy the body over
		io.CopyBuffer(w, res.Body, nil)
	}

	return res.StatusCode, nil
}

func getServiceName(urlValue string) string {
	var serviceName string
	forward := "/function/"
	if startsWith(urlValue, forward) {
		serviceName = urlValue[len(forward):]
	}
	return serviceName
}

func startsWith(value, token string) bool {
	return len(value) > len(token) && strings.Index(value, token) == 0
}
<commit_msg>Proxy fix - copy request headers into upstream<commit_after>package handlers

import (
	"context"
	"io"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/openfaas/faas/gateway/metrics"
	"github.com/openfaas/faas/gateway/types"
	"github.com/prometheus/client_golang/prometheus"
)

// MakeForwardingProxyHandler create a handler which forwards HTTP requests
func MakeForwardingProxyHandler(proxy *types.HTTPClientReverseProxy, metrics *metrics.MetricOptions) http.HandlerFunc {
	baseURL := proxy.BaseURL.String()
	if strings.HasSuffix(baseURL, "/") {
		baseURL = baseURL[0 : len(baseURL)-1]
	}

	return func(w http.ResponseWriter, r *http.Request) {

		requestURL := r.URL.String()
		serviceName := getServiceName(requestURL)

		log.Printf("> Forwarding [%s] to %s", r.Method, requestURL)

		start := time.Now()

		statusCode, err := forwardRequest(w, r, proxy.Client, baseURL, requestURL, proxy.Timeout)

		if err != nil {
			log.Printf("error with upstream request to: %s, %s\n", requestURL, err.Error())
		}

		seconds := time.Since(start).Seconds()
		log.Printf("< [%s] - %d took %f seconds\n", r.URL.String(),
			statusCode, seconds)

		if len(serviceName) > 0 {
			metrics.GatewayFunctionsHistogram.
				WithLabelValues(serviceName).
				Observe(seconds)

			code := strconv.Itoa(statusCode)

			metrics.GatewayFunctionInvocation.
				With(prometheus.Labels{"function_name": serviceName, "code": code}).
				Inc()
		}

	}
}

func forwardRequest(w http.ResponseWriter, r *http.Request, proxyClient *http.Client, baseURL string, requestURL string, timeout time.Duration) (int, error) {

	upstreamReq, _ := http.NewRequest(r.Method, baseURL+requestURL, nil)
	copyHeaders(upstreamReq.Header, &r.Header)

	upstreamReq.Header["X-Forwarded-For"] = []string{r.RemoteAddr}

	if r.Body != nil {
		defer r.Body.Close()
		upstreamReq.Body = r.Body
	}

	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	res, resErr := proxyClient.Do(upstreamReq.WithContext(ctx))
	if resErr != nil {
		badStatus := http.StatusBadGateway
		w.WriteHeader(badStatus)
		return badStatus, resErr
	}

	if res.Body != nil {
		defer res.Body.Close()
	}

	copyHeaders(w.Header(), &res.Header)

	// Write status code
	w.WriteHeader(res.StatusCode)

	if res.Body != nil {
		// Copy the body over
		io.CopyBuffer(w, res.Body, nil)
	}

	return res.StatusCode, nil
}

func copyHeaders(destination http.Header, source *http.Header) {
	for k, v := range *source {
		vClone := make([]string, len(v))
		copy(vClone, v)
		(destination)[k] = vClone
	}
}

func getServiceName(urlValue string) string {
	var serviceName string
	forward := "/function/"
	if startsWith(urlValue, forward) {
		serviceName = urlValue[len(forward):]
	}
	return serviceName
}

func startsWith(value, token string) bool {
	return len(value) > len(token) && strings.Index(value, token) == 0
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"errors"
	"fmt"
	"strings"

	"github.com/fatih/color"
	"github.com/minio/cli"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/pkg/console"
)

var adminPolicyUpdateCmd = cli.Command{
	Name:         "update",
	Usage:        "Attach new IAM policy to a user or group",
	Action:       mainAdminPolicyUpdate,
	OnUsageError: onUsageError,
	Before:       setGlobalsFromContext,
	Flags:        globalFlags,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} TARGET POLICYNAME [ user=username1 | group=groupname1 ]

POLICYNAME:
  Name of the policy on the MinIO server.

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Add the "diagnostics" policy for user "james".
     {{.Prompt}} {{.HelpName}} myminio diagnostics user=james

  2. add the "diagnostics" policy for group "auditors".
     {{.Prompt}} {{.HelpName}} myminio diagnostics group=auditors
`,
}

func checkAdminPolicyUpdateSyntax(ctx *cli.Context) {
	if len(ctx.Args()) != 3 {
		cli.ShowCommandHelpAndExit(ctx, "update", 1) // last argument is exit code
	}
}

func updateCannedPolicies(existingPolicies, policiesToAdd string) (string, error) {
	policiesToAdd = strings.TrimSpace(policiesToAdd)
	if policiesToAdd == "" {
		return "", errors.New("empty policy name is unsupported")
	}
	var updatedPolicies []string
	if existingPolicies != "" {
		updatedPolicies = strings.Split(existingPolicies, ",")
	}

	for _, p1 := range strings.Split(policiesToAdd, ",") {
		found := false
		p1 = strings.TrimSpace(p1)
		for _, p2 := range updatedPolicies {
			if p1 == p2 {
				found = true
				break
			}
		}
		if found {
			return "", fmt.Errorf("policy `%s` already exists", p1)
		}
		updatedPolicies = append(updatedPolicies, p1)
	}

	return strings.Join(updatedPolicies, ","), nil
}

// mainAdminPolicyUpdate is the handler for "mc admin policy update" command.
func mainAdminPolicyUpdate(ctx *cli.Context) error {
	checkAdminPolicyUpdateSyntax(ctx)

	console.SetColor("PolicyMessage", color.New(color.FgGreen))
	console.SetColor("Policy", color.New(color.FgBlue))

	// Get the alias parameter from cli
	args := ctx.Args()
	aliasedURL := args.Get(0)
	policiesToAdd := args.Get(1)
	entityArg := args.Get(2)

	userOrGroup, isGroup, e1 := parseEntityArg(entityArg)
	fatalIf(probe.NewError(e1).Trace(args...), "Bad last argument")

	// Create a new MinIO Admin Client
	client, err := newAdminClient(aliasedURL)
	fatalIf(err, "Unable to initialize admin connection.")

	var existingPolicies string

	if !isGroup {
		userInfo, e := client.GetUserInfo(globalContext, userOrGroup)
		fatalIf(probe.NewError(e).Trace(args...), "Unable to get user policy info")
		existingPolicies = userInfo.PolicyName
	} else {
		groupInfo, e := client.GetGroupDescription(globalContext, userOrGroup)
		fatalIf(probe.NewError(e).Trace(args...), "Unable to get group policy info")
		existingPolicies = groupInfo.Policy
	}

	updatedPolicies, e := updateCannedPolicies(existingPolicies, policiesToAdd)
	if err != nil {
		fatalIf(probe.NewError(e).Trace(args...), "Unable to update the policy")
	}

	e = client.SetPolicy(globalContext, updatedPolicies, userOrGroup, isGroup)
	if e == nil {
		printMsg(userPolicyMessage{
			op:          "update",
			Policy:      policiesToAdd,
			UserOrGroup: userOrGroup,
			IsGroup:     isGroup,
		})
	} else {
		fatalIf(probe.NewError(e), "Unable to unset the policy")
	}
	return nil
}
<commit_msg>typo: Error out if the policy is already set in policy update cmd (#3931)<commit_after>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"errors"
	"fmt"
	"strings"

	"github.com/fatih/color"
	"github.com/minio/cli"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/pkg/console"
)

var adminPolicyUpdateCmd = cli.Command{
	Name:         "update",
	Usage:        "Attach new IAM policy to a user or group",
	Action:       mainAdminPolicyUpdate,
	OnUsageError: onUsageError,
	Before:       setGlobalsFromContext,
	Flags:        globalFlags,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} TARGET POLICYNAME [ user=username1 | group=groupname1 ]

POLICYNAME:
  Name of the policy on the MinIO server.

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Add the "diagnostics" policy for user "james".
     {{.Prompt}} {{.HelpName}} myminio diagnostics user=james

  2. Add the "diagnostics" policy for group "auditors".
     {{.Prompt}} {{.HelpName}} myminio diagnostics group=auditors
`,
}

func checkAdminPolicyUpdateSyntax(ctx *cli.Context) {
	if len(ctx.Args()) != 3 {
		cli.ShowCommandHelpAndExit(ctx, "update", 1) // last argument is exit code
	}
}

func updateCannedPolicies(existingPolicies, policiesToAdd string) (string, error) {
	policiesToAdd = strings.TrimSpace(policiesToAdd)
	if policiesToAdd == "" {
		return "", errors.New("empty policy name is unsupported")
	}
	var updatedPolicies []string
	if existingPolicies != "" {
		updatedPolicies = strings.Split(existingPolicies, ",")
	}

	for _, p1 := range strings.Split(policiesToAdd, ",") {
		found := false
		p1 = strings.TrimSpace(p1)
		for _, p2 := range updatedPolicies {
			if p1 == p2 {
				found = true
				break
			}
		}
		if found {
			return "", fmt.Errorf("policy `%s` already exists", p1)
		}
		updatedPolicies = append(updatedPolicies, p1)
	}

	return strings.Join(updatedPolicies, ","), nil
}

// mainAdminPolicyUpdate is the handler for "mc admin policy update" command.
func mainAdminPolicyUpdate(ctx *cli.Context) error {
	checkAdminPolicyUpdateSyntax(ctx)

	console.SetColor("PolicyMessage", color.New(color.FgGreen))
	console.SetColor("Policy", color.New(color.FgBlue))

	// Get the alias parameter from cli
	args := ctx.Args()
	aliasedURL := args.Get(0)
	policiesToAdd := args.Get(1)
	entityArg := args.Get(2)

	userOrGroup, isGroup, e1 := parseEntityArg(entityArg)
	fatalIf(probe.NewError(e1).Trace(args...), "Bad last argument")

	// Create a new MinIO Admin Client
	client, err := newAdminClient(aliasedURL)
	fatalIf(err, "Unable to initialize admin connection.")

	var existingPolicies string

	if !isGroup {
		userInfo, e := client.GetUserInfo(globalContext, userOrGroup)
		fatalIf(probe.NewError(e).Trace(args...), "Unable to get user policy info")
		existingPolicies = userInfo.PolicyName
	} else {
		groupInfo, e := client.GetGroupDescription(globalContext, userOrGroup)
		fatalIf(probe.NewError(e).Trace(args...), "Unable to get group policy info")
		existingPolicies = groupInfo.Policy
	}

	updatedPolicies, e := updateCannedPolicies(existingPolicies, policiesToAdd)
	if e != nil {
		fatalIf(probe.NewError(e).Trace(args...), "Unable to update the policy")
	}

	e = client.SetPolicy(globalContext, updatedPolicies, userOrGroup, isGroup)
	if e == nil {
		printMsg(userPolicyMessage{
			op:          "update",
			Policy:      policiesToAdd,
			UserOrGroup: userOrGroup,
			IsGroup:     isGroup,
		})
	} else {
		fatalIf(probe.NewError(e), "Unable to unset the policy")
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"path"
	"syscall"
)

type stateType struct {
	processedInodes map[uint64]bool
}

func walk(rootDirName, dirName, objectsDir string) error {
	var state stateType
	state.processedInodes = make(map[uint64]bool)
	return state.walk(rootDirName, dirName, objectsDir)
}

func (state *stateType) walk(rootDirName, dirName, objectsDir string) error {
	file, err := os.Open(path.Join(rootDirName, dirName))
	if err != nil {
		return err
	}
	names, err := file.Readdirnames(-1)
	file.Close()
	if err != nil {
		return err
	}
	for _, name := range names {
		if dirName == "/" && name == ".subd" {
			continue
		}
		filename := path.Join(dirName, name)
		pathname := path.Join(rootDirName, filename)
		var stat syscall.Stat_t
		err := syscall.Lstat(pathname, &stat)
		if err != nil {
			return err
		}
		if stat.Mode&syscall.S_IFMT == syscall.S_IFDIR {
			err = state.walk(rootDirName, filename, objectsDir)
			if err == nil {
				err = os.Remove(pathname)
			}
		} else if stat.Mode&syscall.S_IFMT == syscall.S_IFREG {
			err = state.handleFile(pathname, stat.Ino, objectsDir)
		} else {
			err = os.RemoveAll(pathname)
		}
		if err != nil {
			return err
		}
	}
	return nil
}

func (state *stateType) handleFile(pathname string, inum uint64,
	objectsDir string) error {
	if state.processedInodes[inum] {
		return os.Remove(pathname)
	}
	state.processedInodes[inum] = true
	return convertToObject(pathname, objectsDir)
}
<commit_msg>Change fs2objectcache to use struct{} for map values.<commit_after>package main

import (
	"os"
	"path"
	"syscall"
)

type stateType struct {
	processedInodes map[uint64]struct{}
}

func walk(rootDirName, dirName, objectsDir string) error {
	var state stateType
	state.processedInodes = make(map[uint64]struct{})
	return state.walk(rootDirName, dirName, objectsDir)
}

func (state *stateType) walk(rootDirName, dirName, objectsDir string) error {
	file, err := os.Open(path.Join(rootDirName, dirName))
	if err != nil {
		return err
	}
	names, err := file.Readdirnames(-1)
	file.Close()
	if err != nil {
		return err
	}
	for _, name := range names {
		if dirName == "/" && name == ".subd" {
			continue
		}
		filename := path.Join(dirName, name)
		pathname := path.Join(rootDirName, filename)
		var stat syscall.Stat_t
		err := syscall.Lstat(pathname, &stat)
		if err != nil {
			return err
		}
		if stat.Mode&syscall.S_IFMT == syscall.S_IFDIR {
			err = state.walk(rootDirName, filename, objectsDir)
			if err == nil {
				err = os.Remove(pathname)
			}
		} else if stat.Mode&syscall.S_IFMT == syscall.S_IFREG {
			err = state.handleFile(pathname, stat.Ino, objectsDir)
		} else {
			err = os.RemoveAll(pathname)
		}
		if err != nil {
			return err
		}
	}
	return nil
}

func (state *stateType) handleFile(pathname string, inum uint64,
	objectsDir string) error {
	if _, ok := state.processedInodes[inum]; ok {
		return os.Remove(pathname)
	}
	state.processedInodes[inum] = struct{}{}
	return convertToObject(pathname, objectsDir)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"regexp"

	"github.com/Maki-Daisuke/go-argvreader"
	"github.com/Maki-Daisuke/go-lines"
	"github.com/Maki-Daisuke/go-triematcher"
	"github.com/jessevdk/go-flags"
)

var opts struct {
	PkgName string `short:"P" long:"package" default:"main" description:"package name"`
	TagName string `short:"T" long:"tag" default:"" description:"tag name included in the generated functions"`
}

var reId = regexp.MustCompile(`^[0-9a-zA-Z_]+$`)

func main() {
	args, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	if !reId.MatchString(opts.PkgName) {
		fmt.Fprintf(os.Stderr, "Package name must be an identifier, but %q is not\n", opts.PkgName)
		os.Exit(1)
	}
	if opts.TagName != "" && !reId.MatchString(opts.TagName) {
		fmt.Fprintf(os.Stderr, "Tag name must be an identifier, but %q is not\n", opts.TagName)
		os.Exit(1)
	}

	signatures := []string{}

	reader := argvreader.NewReader(args)
	line_chan, err_chan := lines.LinesWithError(reader)
	for line := range line_chan {
		if line == "" {
			continue
		}
		signatures = append(signatures, line)
	}
	err = <-err_chan
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	fmt.Printf("package %s\n\n", opts.PkgName)

	err = triematcher.GenerateMatcher(os.Stdout, opts.TagName, signatures)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<commit_msg>Fix typo.<commit_after>package main

import (
	"fmt"
	"os"
	"regexp"

	"github.com/Maki-Daisuke/go-argvreader"
	"github.com/Maki-Daisuke/go-gentriematcher"
	"github.com/Maki-Daisuke/go-lines"
	"github.com/jessevdk/go-flags"
)

var opts struct {
	PkgName string `short:"P" long:"package" default:"main" description:"package name"`
	TagName string `short:"T" long:"tag" default:"" description:"tag name included in the generated functions"`
}

var reId = regexp.MustCompile(`^[0-9a-zA-Z_]+$`)

func main() {
	args, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	if !reId.MatchString(opts.PkgName) {
		fmt.Fprintf(os.Stderr, "Package name must be an identifier, but %q is not\n", opts.PkgName)
		os.Exit(1)
	}
	if opts.TagName != "" && !reId.MatchString(opts.TagName) {
		fmt.Fprintf(os.Stderr, "Tag name must be an identifier, but %q is not\n", opts.TagName)
		os.Exit(1)
	}

	signatures := []string{}

	reader := argvreader.NewReader(args)
	line_chan, err_chan := lines.LinesWithError(reader)
	for line := range line_chan {
		if line == "" {
			continue
		}
		signatures = append(signatures, line)
	}
	err = <-err_chan
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	fmt.Printf("package %s\n\n", opts.PkgName)

	err = triematcher.GenerateMatcher(os.Stdout, opts.TagName, signatures)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"fmt"
	"io"
	"os"
	"time"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"launchpad.net/gnuflag"
)

const (
	notset           = "<not set>"
	filenameTemplate = "juju-backup-%04d%02d%02d-%02d%02d%02d.tar.gz"
)

const createDoc = `
"create" requests that juju create a backup of its state and print the
backup's unique ID.  You may provide a note to associate with the backup.

The backup archive and associated metadata are stored in juju and
will be lost when the environment is destroyed.
`

// CreateCommand is the sub-command for creating a new backup.
type CreateCommand struct {
	CommandBase
	// Quiet indicates that the full metadata should not be dumped.
	Quiet bool
	// Download indicates that the backups archive should be downloaded.
	Download bool
	// Filename is where the backup should be downloaded.
	Filename string
	// Notes is the custom message to associated with the new backup.
	Notes string
}

// Info implements Command.Info.
func (c *CreateCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "create",
		Args:    "[<notes>]",
		Purpose: "create a backup",
		Doc:     createDoc,
	}
}

// SetFlags implements Command.SetFlags.
func (c *CreateCommand) SetFlags(f *gnuflag.FlagSet) {
	f.BoolVar(&c.Quiet, "quiet", false, "do not print the metadata")
	f.BoolVar(&c.Download, "download", false, "download the archive")
	f.StringVar(&c.Filename, "filename", notset, "download to this file")
}

// Init implements Command.Init.
func (c *CreateCommand) Init(args []string) error {
	notes, err := cmd.ZeroOrOneArgs(args)
	if err != nil {
		return err
	}
	c.Notes = notes
	return nil
}

// Run implements Command.Run.
func (c *CreateCommand) Run(ctx *cmd.Context) error {
	client, err := c.NewAPIClient()
	if err != nil {
		return errors.Trace(err)
	}
	defer client.Close()

	result, err := client.Create(c.Notes)
	if err != nil {
		return errors.Trace(err)
	}

	if !c.Quiet {
		c.dumpMetadata(ctx, result)
	}

	fmt.Fprintln(ctx.Stdout, result.ID)

	// Handle download.
	filename := c.decideFilename(ctx, c.Filename, result.Started)
	if filename != "" {
		if err := c.download(ctx, client, result.ID, filename); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

func (c *CreateCommand) decideFilename(ctx *cmd.Context, filename string, timestamp time.Time) string {
	if filename == "" {
		fmt.Fprintln(ctx.Stderr, "missing filename")
	} else if c.Filename == notset {
		if c.Download {
			y, m, d := timestamp.Date()
			H, M, S := timestamp.Clock()
			filename = fmt.Sprintf(filenameTemplate, y, m, d, H, M, S)
		} else {
			filename = ""
		}
	}
	return filename
}

func (c *CreateCommand) download(ctx *cmd.Context, client APIClient, id string, filename string) error {
	fmt.Fprintln(ctx.Stdout, "downloading to "+filename)

	archive, err := client.Download(id)
	if err != nil {
		return errors.Trace(err)
	}
	defer archive.Close()

	outfile, err := os.Create(filename)
	if err != nil {
		return errors.Trace(err)
	}
	defer outfile.Close()

	_, err = io.Copy(outfile, archive)
	if err != nil {
		return errors.Trace(err)
	}

	return nil
}
<commit_msg>Drop superfluous if statement.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"fmt"
	"io"
	"os"
	"time"

	"github.com/juju/cmd"
	"github.com/juju/errors"
	"launchpad.net/gnuflag"
)

const (
	notset           = "<not set>"
	filenameTemplate = "juju-backup-%04d%02d%02d-%02d%02d%02d.tar.gz"
)

const createDoc = `
"create" requests that juju create a backup of its state and print the
backup's unique ID.  You may provide a note to associate with the backup.

The backup archive and associated metadata are stored in juju and
will be lost when the environment is destroyed.
`

// CreateCommand is the sub-command for creating a new backup.
type CreateCommand struct {
	CommandBase
	// Quiet indicates that the full metadata should not be dumped.
	Quiet bool
	// Download indicates that the backups archive should be downloaded.
	Download bool
	// Filename is where the backup should be downloaded.
	Filename string
	// Notes is the custom message to associated with the new backup.
	Notes string
}

// Info implements Command.Info.
func (c *CreateCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "create",
		Args:    "[<notes>]",
		Purpose: "create a backup",
		Doc:     createDoc,
	}
}

// SetFlags implements Command.SetFlags.
func (c *CreateCommand) SetFlags(f *gnuflag.FlagSet) {
	f.BoolVar(&c.Quiet, "quiet", false, "do not print the metadata")
	f.BoolVar(&c.Download, "download", false, "download the archive")
	f.StringVar(&c.Filename, "filename", notset, "download to this file")
}

// Init implements Command.Init.
func (c *CreateCommand) Init(args []string) error {
	notes, err := cmd.ZeroOrOneArgs(args)
	if err != nil {
		return err
	}
	c.Notes = notes
	return nil
}

// Run implements Command.Run.
func (c *CreateCommand) Run(ctx *cmd.Context) error {
	client, err := c.NewAPIClient()
	if err != nil {
		return errors.Trace(err)
	}
	defer client.Close()

	result, err := client.Create(c.Notes)
	if err != nil {
		return errors.Trace(err)
	}

	if !c.Quiet {
		c.dumpMetadata(ctx, result)
	}

	fmt.Fprintln(ctx.Stdout, result.ID)

	// Handle download.
	filename := c.decideFilename(ctx, c.Filename, result.Started)
	if filename != "" {
		if err := c.download(ctx, client, result.ID, filename); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

func (c *CreateCommand) decideFilename(ctx *cmd.Context, filename string, timestamp time.Time) string {
	if filename == "" {
		fmt.Fprintln(ctx.Stderr, "missing filename")
	} else if c.Filename == notset {
		if c.Download {
			y, m, d := timestamp.Date()
			H, M, S := timestamp.Clock()
			filename = fmt.Sprintf(filenameTemplate, y, m, d, H, M, S)
		} else {
			filename = ""
		}
	}
	return filename
}

func (c *CreateCommand) download(ctx *cmd.Context, client APIClient, id string, filename string) error {
	fmt.Fprintln(ctx.Stdout, "downloading to "+filename)

	archive, err := client.Download(id)
	if err != nil {
		return errors.Trace(err)
	}
	defer archive.Close()

	outfile, err := os.Create(filename)
	if err != nil {
		return errors.Trace(err)
	}
	defer outfile.Close()

	_, err = io.Copy(outfile, archive)
	return errors.Trace(err)
}
<|endoftext|>


<commit_before>// Copyright 2014-2015 The DevMine authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"database/sql"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	_ "github.com/lib/pq"

	"github.com/DevMine/crawld/config"
	"github.com/DevMine/crawld/crawlers"
	"github.com/DevMine/crawld/errbag"
	"github.com/DevMine/crawld/repo"
	"github.com/DevMine/crawld/tar"
)

// extend this structure later if required but for now the repository id sufficient
type dbRepo struct {
	repo.Repo
	id uint64
}

// channel used to communicate repositories IDs
var idChan chan uint64

func crawlingWorker(cs []crawlers.Crawler, crawlingInterval time.Duration) {
	for {
		var wg sync.WaitGroup

		wg.Add(len(cs))
		for _, c := range cs {
			glog.Infof("starting a goroutine for the %v crawler\n", reflect.TypeOf(c))
			go func(c crawlers.Crawler) {
				defer wg.Done()
				c.Crawl()
			}(c)
		}

		wg.Wait()

		glog.Infof("waiting for %v before re-starting the crawlers.\n", crawlingInterval)
		<-time.After(crawlingInterval)
	}
}

func repoWorker(db *sql.DB, cfg *config.Config, startId uint64, errBag *errbag.ErrBag) {

	fetchInterval, err := time.ParseDuration(cfg.FetchTimeInterval)
	if err != nil {
		fatal(err)
	}

	clone := func(r repo.Repo) error {
		glog.Infof("cloning %s into %s\n", r.URL(), r.AbsPath())
		if err := r.Clone(); err != nil {
			glog.Errorf("impossible to clone %s in %s ("+err.Error()+") skipping", r.URL(), r.AbsPath())
			errBag.Record(err)
			return err
		}
		return nil
	}

	update := func(r repo.Repo) error {
		glog.Infof("updating %s\n", r.AbsPath())
		if err := r.Update(); err != nil {
			glog.Warningf("impossible to update %s ("+err.Error()+")", r.AbsPath())
			errBag.Record(err)

			// we just want to skip on a network error
			if err == repo.ErrNetwork {
				return err
			}

			// delete and reclone then
			glog.Infof("attempting to re-clone %s", r.AbsPath())
			if err2 := os.RemoveAll(r.AbsPath()); err2 != nil {
				glog.Errorf("cannot remove %s("+err2.Error()+")", r.AbsPath())
				errBag.Record(err)
				return err
			}
			return clone(r)
		}
		return nil
	}

	createArchive := func(path string) {
		if err := tar.CreateInPlace(path); err != nil {
			glog.Error("impossible to create tar archive (" + path + ".tar ): " +
				err.Error())
			errBag.Record(err)
		}
	}

	for {
		glog.Info("starting the repositories fetcher")
		repos, err := getAllRepos(db, startId, cfg.FetchLanguages, cfg.CloneDir)
		if err != nil {
			fatal(err)
		}
		// next time, we want to get all repos from the first one
		startId = 0

		tasks := make(chan dbRepo, len(repos))
		var wg sync.WaitGroup

		for _, r := range repos {
			tasks <- r
		}
		// we don't want any routine to add new tasks in the queue now
		// if we don't close the channel now, the goroutines processing the
		// tasks will wait forever for new tasks and never return
		close(tasks)

		for w := uint(0); w < cfg.MaxFetcherWorkers; w++ {
			wg.Add(1)
			go func() {
				for r := range tasks {
					// if we have a tar archive, we need to extract it
					archive := r.AbsPath() + ".tar"
					if _, err = os.Stat(archive); err == nil {
						if err = tar.ExtractInPlace(archive); err != nil {
							glog.Warning("impossible to extract the tar archive (" + archive + ")" +
								", cannot update the repository: " + err.Error())
							// attempt to remove the eventual mess
							_ = os.Remove(archive)
							_ = os.RemoveAll(r.AbsPath())
						}
					}

					if _, err := os.Stat(r.AbsPath()); os.IsNotExist(err) || isDirEmpty(r.AbsPath()) {
						if err = clone(r); err != nil {
							continue
						}
					} else {
						if err = update(r); err != nil {
							continue
						}
					}

					if cfg.TarRepos {
						createArchive(r.AbsPath())
					}

					if err = r.Cleanup(); err != nil {
						glog.Warning(err)
					}

					// notify we're done with this repository
					idChan <- r.id
				}
				wg.Done()
			}()
		}

		wg.Wait()

		glog.Infof("waiting for %v before re-starting the fetcher.\n", fetchInterval)
		<-time.After(fetchInterval)
	}
}

func isDirEmpty(path string) bool {
	fis, err := ioutil.ReadDir(path)
	if err != nil {
		return false
	}

	return len(fis) == 0
}

func getAllRepos(db *sql.DB, startId uint64, langs []string, basePath string) ([]dbRepo, error) {
	inClause := fmt.Sprintf("WHERE id >= %d", startId)
	if langs != nil && len(langs) > 0 {
		// Quote languages.
		for idx, val := range langs {
			langs[idx] = "'" + val + "'"
		}
		inClause += " AND LOWER(primary_language) IN (" + strings.Join(langs, ",") + ")"
	}

	rows, err := db.Query("SELECT id, vcs, clone_path, clone_url FROM repositories " + inClause + " ORDER BY id")
	if err != nil {
		glog.Error(err)
		return nil, err
	}
	defer rows.Close()

	var repos []dbRepo

	for rows.Next() {
		var vcs, clonePath, cloneURL string
		var id uint64
		if err := rows.Scan(&id, &vcs, &clonePath, &cloneURL); err != nil {
			glog.Error(err)
			continue
		}

		var newRepo repo.Repo
		var err error
		newRepo, err = repo.New(vcs, filepath.Join(basePath, clonePath), cloneURL)
		if err != nil {
			glog.Error(err)
			continue
		}

		repos = append(repos, dbRepo{Repo: newRepo, id: id})
	}

	return repos, nil
}

func checkCloneDir(cloneDir string) error {
	// check if clone path exists
	if fi, err := os.Stat(cloneDir); err == nil {
		glog.Error(err)
		return err
	} else if !fi.IsDir() {
		err = errors.New("clone path must be a directory")
		glog.Error(err)
		return err
	}

	// check if clone path is writable
	// note: since the directory already exists, then the file perm param is
	// useless
	file, err := os.OpenFile(cloneDir, os.O_RDWR, 0770)
	if err != nil {
		err = errors.New("clone path must be writable")
		glog.Error(err)
		return err
	}
	file.Close()

	return nil
}

func openDBSession(cfg config.DatabaseConfig) (*sql.DB, error) {
	dbURL := fmt.Sprintf(
		"user='%s' password='%s' host='%s' port=%d dbname='%s' sslmode='%s'",
		cfg.UserName, cfg.Password, cfg.HostName, cfg.Port, cfg.DBName, cfg.SSLMode)

	return sql.Open("postgres", dbURL)
}

func fatal(a ...interface{}) {
	glog.Error(a)
	os.Exit(1)
}

func main() {
	configPath := flag.String("c", "", "configuration file")
	disableCrawlers := flag.Bool("disable-crawlers", false, "disable the data crawlers")
	disableFetcher := flag.Bool("disable-fetcher", false, "disable the repositories fetcher")
	flag.Parse()

	// Make sure we finish writing logs before exiting.
	defer glog.Flush()

	if len(*configPath) == 0 {
		fatal("no configuration specified")
	}

	cfg, err := config.ReadConfig(*configPath)
	if err != nil {
		fatal(err)
	}

	db, err := openDBSession(cfg.Database)
	if err != nil {
		fatal(err)
	}
	defer db.Close()

	var cs []crawlers.Crawler

	for _, crawlerConfig := range cfg.Crawlers {
		c, err := crawlers.New(crawlerConfig, db)
		if err != nil {
			fatal(err)
		}

		cs = append(cs, c)
	}

	crawlingInterval, err := time.ParseDuration(cfg.CrawlingTimeInterval)
	if err != nil {
		fatal(err)
	}

	var wg sync.WaitGroup

	// start the crawling worker
	if !*disableCrawlers {
		wg.Add(1)
		go crawlingWorker(cs, crawlingInterval)
	}

	// start the repo puller worker
	if !*disableFetcher {
		errBag, err := errbag.New(cfg.ThrottlerWaitTime, cfg.SlidingWindowSize, cfg.LeakInterval)
		if err != nil {
			glog.Error("impossible to start the repositories fetcher")
			return
		}
		errBag.Inflate()

		var startId uint64
		lastFetchedIdFile := path.Join(cfg.CloneDir, "last_fetched_id")
		if bs, err := ioutil.ReadFile(lastFetchedIdFile); len(bs) != 0 && err == nil {
			if startId, err = strconv.ParseUint(string(bs), 10, 64); err != nil {
				glog.Warning("cannot convert (" + string(bs) + ") to a repository id, starting from 0...")
				startId = 0
			}
		} else {
			glog.Warning("cannot get last fetched repository id, starting from 0...")
			startId = 0
		}

		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt, os.Kill)

		idChan = make(chan uint64)

		// this routines writes the last processed repository id in a file, getting it from idChan
		go func() {
			f, err := os.OpenFile(lastFetchedIdFile, os.O_WRONLY|os.O_CREATE, 0644)
			if err != nil {
				glog.Fatal("cannot open file for writing (" + lastFetchedIdFile + "): " + err.Error())
			}

			// we want to make sure we close the file and do some housekeeping on interruption
			go func() {
				<-c
				fmt.Fprintln(os.Stderr, "caught signal, exiting now...")
				f.Sync()
				f.Close()
				errBag.Deflate()
				os.Exit(0)
			}()

			for id, ok := <-idChan; ok; id, ok = <-idChan {
				if _, err := f.Seek(0, 0); err != nil {
					glog.Warning("could not write ID to file:", id)
				} else {
					// pad with 0 up to 20 because the largest unsigned integer
					// of 64 bit fits in 20 digits in decimal format
					fmt.Fprintf(f, "%020d", id)
				}
			}
		}()

		wg.Add(1)
		go repoWorker(db, cfg, startId, errBag)
	}

	// wait until the cows come home saint
	wg.Wait()
}
<commit_msg>repo: use callback function with errbag throttler<commit_after>// Copyright 2014-2015 The DevMine authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"database/sql"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"path"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	_ "github.com/lib/pq"

	"github.com/DevMine/crawld/config"
	"github.com/DevMine/crawld/crawlers"
	"github.com/DevMine/crawld/errbag"
	"github.com/DevMine/crawld/repo"
	"github.com/DevMine/crawld/tar"
)

// extend this structure later if required but for now the repository id sufficient
type dbRepo struct {
	repo.Repo
	id uint64
}

// channel used to communicate repositories IDs
var idChan chan uint64

func crawlingWorker(cs []crawlers.Crawler, crawlingInterval time.Duration) {
	for {
		var wg sync.WaitGroup

		wg.Add(len(cs))
		for _, c := range cs {
			glog.Infof("starting a goroutine for the %v crawler\n", reflect.TypeOf(c))
			go func(c crawlers.Crawler) {
				defer wg.Done()
				c.Crawl()
			}(c)
		}

		wg.Wait()

		glog.Infof("waiting for %v before re-starting the crawlers.\n", crawlingInterval)
		<-time.After(crawlingInterval)
	}
}

func repoWorker(db *sql.DB, cfg *config.Config, startId uint64, errBag *errbag.ErrBag) {

	fetchInterval, err := time.ParseDuration(cfg.FetchTimeInterval)
	if err != nil {
		fatal(err)
	}

	callback := func(status errbag.Status) {
		if status.State == errbag.StatusThrottling {
			glog.Info("too many errors received; waiting for ", status.WaitTime, " seconds before resuming")
		}
	}

	clone := func(r repo.Repo) error {
		glog.Infof("cloning %s into %s\n", r.URL(), r.AbsPath())
		if err := r.Clone(); err != nil {
			glog.Errorf("impossible to clone %s in %s ("+err.Error()+") skipping", r.URL(), r.AbsPath())
			errBag.Record(err, callback)
			return err
		}
		return nil
	}

	update := func(r repo.Repo) error {
		glog.Infof("updating %s\n", r.AbsPath())
		if err := r.Update(); err != nil {
			glog.Warningf("impossible to update %s ("+err.Error()+")", r.AbsPath())
			errBag.Record(err, callback)

			// we just want to skip on a network error
			if err == repo.ErrNetwork {
				return err
			}

			// delete and reclone then
			glog.Infof("attempting to re-clone %s", r.AbsPath())
			if err2 := os.RemoveAll(r.AbsPath()); err2 != nil {
				glog.Errorf("cannot remove %s("+err2.Error()+")", r.AbsPath())
				errBag.Record(err, callback)
				return err
			}
			return clone(r)
		}
		return nil
	}

	createArchive := func(path string) {
		if err := tar.CreateInPlace(path); err != nil {
			glog.Error("impossible to create tar archive (" + path + ".tar ): " +
				err.Error())
			errBag.Record(err, callback)
		}
	}

	for {
		glog.Info("starting the repositories fetcher")
		repos, err := getAllRepos(db, startId, cfg.FetchLanguages, cfg.CloneDir)
		if err != nil {
			fatal(err)
		}
		// next time, we want to get all repos from the first one
		startId = 0

		tasks := make(chan dbRepo, len(repos))
		var wg sync.WaitGroup

		for _, r := range repos {
			tasks <- r
		}
		// we don't want any routine to add new tasks in the queue now
		// if we don't close the channel now, the goroutines processing the
		// tasks will wait forever for new tasks and never return
		close(tasks)

		for w := uint(0); w < cfg.MaxFetcherWorkers; w++ {
			wg.Add(1)
			go func() {
				for r := range tasks {
					// if we have a tar archive, we need to extract it
					archive := r.AbsPath() + ".tar"
					if _, err = os.Stat(archive); err == nil {
						if err = tar.ExtractInPlace(archive); err != nil {
							glog.Warning("impossible to extract the tar archive (" + archive + ")" +
								", cannot update the repository: " + err.Error())
							// attempt to remove the eventual mess
							_ = os.Remove(archive)
							_ = os.RemoveAll(r.AbsPath())
						}
					}

					if _, err := os.Stat(r.AbsPath()); os.IsNotExist(err) || isDirEmpty(r.AbsPath()) {
						if err = clone(r); err != nil {
							continue
						}
					} else {
						if err = update(r); err != nil {
							continue
						}
					}

					if cfg.TarRepos {
						createArchive(r.AbsPath())
					}

					if err = r.Cleanup(); err != nil {
						glog.Warning(err)
					}

					// notify we're done with this repository
					idChan <- r.id
				}
				wg.Done()
			}()
		}

		wg.Wait()

		glog.Infof("waiting for %v before re-starting the fetcher.\n", fetchInterval)
		<-time.After(fetchInterval)
	}
}

func isDirEmpty(path string) bool {
	fis, err := ioutil.ReadDir(path)
	if err != nil {
		return false
	}

	return len(fis) == 0
}

func getAllRepos(db *sql.DB, startId uint64, langs []string, basePath string) ([]dbRepo, error) {
	inClause := fmt.Sprintf("WHERE id >= %d", startId)
	if langs != nil && len(langs) > 0 {
		// Quote languages.
		for idx, val := range langs {
			langs[idx] = "'" + val + "'"
		}
		inClause += " AND LOWER(primary_language) IN (" + strings.Join(langs, ",") + ")"
	}

	rows, err := db.Query("SELECT id, vcs, clone_path, clone_url FROM repositories " + inClause + " ORDER BY id")
	if err != nil {
		glog.Error(err)
		return nil, err
	}
	defer rows.Close()

	var repos []dbRepo

	for rows.Next() {
		var vcs, clonePath, cloneURL string
		var id uint64
		if err := rows.Scan(&id, &vcs, &clonePath, &cloneURL); err != nil {
			glog.Error(err)
			continue
		}

		var newRepo repo.Repo
		var err error
		newRepo, err = repo.New(vcs, filepath.Join(basePath, clonePath), cloneURL)
		if err != nil {
			glog.Error(err)
			continue
		}

		repos = append(repos, dbRepo{Repo: newRepo, id: id})
	}

	return repos, nil
}

func checkCloneDir(cloneDir string) error {
	// check if clone path exists
	if fi, err := os.Stat(cloneDir); err == nil {
		glog.Error(err)
		return err
	} else if !fi.IsDir() {
		err = errors.New("clone path must be a directory")
		glog.Error(err)
		return err
	}

	// check if clone path is writable
	// note: since the directory already exists, then the file perm param is
	// useless
	file, err := os.OpenFile(cloneDir, os.O_RDWR, 0770)
	if err != nil {
		err = errors.New("clone path must be writable")
		glog.Error(err)
		return err
	}
	file.Close()

	return nil
}

func openDBSession(cfg config.DatabaseConfig) (*sql.DB, error) {
	dbURL := fmt.Sprintf(
		"user='%s' password='%s' host='%s' port=%d dbname='%s' sslmode='%s'",
		cfg.UserName, cfg.Password, cfg.HostName, cfg.Port, cfg.DBName, cfg.SSLMode)

	return sql.Open("postgres", dbURL)
}

func fatal(a ...interface{}) {
	glog.Error(a)
	os.Exit(1)
}

func main() {
	configPath := flag.String("c", "", "configuration file")
	disableCrawlers := flag.Bool("disable-crawlers", false, "disable the data crawlers")
	disableFetcher := flag.Bool("disable-fetcher", false, "disable the repositories fetcher")
	flag.Parse()

	// Make sure we finish writing logs before exiting.
	defer glog.Flush()

	if len(*configPath) == 0 {
		fatal("no configuration specified")
	}

	cfg, err := config.ReadConfig(*configPath)
	if err != nil {
		fatal(err)
	}

	db, err := openDBSession(cfg.Database)
	if err != nil {
		fatal(err)
	}
	defer db.Close()

	var cs []crawlers.Crawler

	for _, crawlerConfig := range cfg.Crawlers {
		c, err := crawlers.New(crawlerConfig, db)
		if err != nil {
			fatal(err)
		}

		cs = append(cs, c)
	}

	crawlingInterval, err := time.ParseDuration(cfg.CrawlingTimeInterval)
	if err != nil {
		fatal(err)
	}

	var wg sync.WaitGroup

	// start the crawling worker
	if !*disableCrawlers {
		wg.Add(1)
		go crawlingWorker(cs, crawlingInterval)
	}

	// start the repo puller worker
	if !*disableFetcher {
		errBag, err := errbag.New(cfg.ThrottlerWaitTime, cfg.SlidingWindowSize, cfg.LeakInterval)
		if err != nil {
			glog.Error("impossible to start the repositories fetcher")
			return
		}
		errBag.Inflate()

		var startId uint64
		lastFetchedIdFile := path.Join(cfg.CloneDir, "last_fetched_id")
		if bs, err := ioutil.ReadFile(lastFetchedIdFile); len(bs) != 0 && err == nil {
			if startId, err = strconv.ParseUint(string(bs), 10, 64); err != nil {
				glog.Warning("cannot convert (" + string(bs) + ") to a repository id, starting from 0...")
				startId = 0
			}
		} else {
			glog.Warning("cannot get last fetched repository id, starting from 0...")
			startId = 0
		}

		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt, os.Kill)

		idChan = make(chan uint64)

		// this routines writes the last processed repository id in a file, getting it from idChan
		go func() {
			f, err := os.OpenFile(lastFetchedIdFile, os.O_WRONLY|os.O_CREATE, 0644)
			if err != nil {
				glog.Fatal("cannot open file for writing (" + lastFetchedIdFile + "): " + err.Error())
			}

			// we want to make sure we close the file and do some housekeeping on interruption
			go func() {
				<-c
				fmt.Fprintln(os.Stderr, "caught signal, exiting now...")
				f.Sync()
				f.Close()
				errBag.Deflate()
				os.Exit(0)
			}()

			for id, ok := <-idChan; ok; id, ok = <-idChan {
				if _, err := f.Seek(0, 0); err != nil {
					glog.Warning("could not write ID to file:", id)
				} else {
					// pad with 0 up to 20 because the largest unsigned integer
					// of 64 bit fits in 20 digits in decimal format
					fmt.Fprintf(f, "%020d", id)
				}
			}
		}()

		wg.Add(1)
		go repoWorker(db, cfg, startId, errBag)
	}

	// wait until the cows come home saint
	wg.Wait()
}
<|endoftext|>