<commit_before>package openshiftapiserver

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"github.com/emicklei/go-restful-swagger12"
	"github.com/golang/glog"

	"k8s.io/apiserver/pkg/admission"
	admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
	genericapiserver "k8s.io/apiserver/pkg/server"
	genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
	cacheddiscovery "k8s.io/client-go/discovery/cached"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/restmapper"
	"k8s.io/client-go/tools/cache"
	"k8s.io/kubernetes/pkg/api/legacyscheme"

	configv1 "github.com/openshift/api/config/v1"
	openshiftcontrolplanev1 "github.com/openshift/api/openshiftcontrolplane/v1"
	"github.com/openshift/library-go/pkg/config/helpers"
	"github.com/openshift/origin/pkg/admission/namespaceconditions"
	"github.com/openshift/origin/pkg/api/legacy"
	originadmission "github.com/openshift/origin/pkg/apiserver/admission"
	"github.com/openshift/origin/pkg/cmd/openshift-apiserver/openshiftapiserver/configprocessing"
	configlatest "github.com/openshift/origin/pkg/cmd/server/apis/config/latest"
	"github.com/openshift/origin/pkg/image/apiserver/registryhostname"
	usercache "github.com/openshift/origin/pkg/user/cache"
	"github.com/openshift/origin/pkg/version"
)

func NewOpenshiftAPIConfig(config *openshiftcontrolplanev1.OpenShiftAPIServerConfig) (*OpenshiftAPIConfig, error) {
	kubeClientConfig, err := helpers.GetKubeClientConfig(config.KubeClientConfig)
	if err != nil {
		return nil, err
	}
	kubeClient, err := kubernetes.NewForConfig(kubeClientConfig)
	if err != nil {
		return nil, err
	}
	kubeInformers := informers.NewSharedInformerFactory(kubeClient, 10*time.Minute)

	openshiftVersion := version.Get()

	backend, policyChecker, err := configprocessing.GetAuditConfig(config.AuditConfig)
	if err != nil {
		return nil, err
	}
	restOptsGetter, err := NewRESTOptionsGetter(config.APIServerArguments, config.StorageConfig)
	if err != nil {
		return nil, err
	}

	genericConfig := genericapiserver.NewRecommendedConfig(legacyscheme.Codecs)
	// Current default values
	//Serializer:                   codecs,
	//ReadWritePort:                443,
	//BuildHandlerChainFunc:        DefaultBuildHandlerChain,
	//HandlerChainWaitGroup:        new(utilwaitgroup.SafeWaitGroup),
	//LegacyAPIGroupPrefixes:       sets.NewString(DefaultLegacyAPIPrefix),
	//DisabledPostStartHooks:       sets.NewString(),
	//HealthzChecks:                []healthz.HealthzChecker{healthz.PingHealthz, healthz.LogHealthz},
	//EnableIndex:                  true,
	//EnableDiscovery:              true,
	//EnableProfiling:              true,
	//EnableMetrics:                true,
	//MaxRequestsInFlight:          400,
	//MaxMutatingRequestsInFlight:  200,
	//RequestTimeout:               time.Duration(60) * time.Second,
	//MinRequestTimeout:            1800,
	//EnableAPIResponseCompression: utilfeature.DefaultFeatureGate.Enabled(features.APIResponseCompression),
	//LongRunningFunc: genericfilters.BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString()),

	// TODO this is actually specific to the kubeapiserver
	//RuleResolver authorizer.RuleResolver
	genericConfig.SharedInformerFactory = kubeInformers
	genericConfig.ClientConfig = kubeClientConfig

	// these are set via options
	//SecureServing *SecureServingInfo
	//Authentication AuthenticationInfo
	//Authorization AuthorizationInfo
	//LoopbackClientConfig *restclient.Config
	// this is set after the options are overlayed to get the authorizer we need.
	//AdmissionControl      admission.Interface
	//ReadWritePort int
	//PublicAddress net.IP

	// these are defaulted sanely during complete
	//DiscoveryAddresses discovery.Addresses

	genericConfig.CorsAllowedOriginList = config.CORSAllowedOrigins
	genericConfig.Version = &openshiftVersion
	// we don't use legacy audit anymore
	genericConfig.LegacyAuditWriter = nil
	genericConfig.AuditBackend = backend
	genericConfig.AuditPolicyChecker = policyChecker
	genericConfig.ExternalAddress = "apiserver.openshift-apiserver.svc"
	genericConfig.BuildHandlerChainFunc = OpenshiftHandlerChain
	genericConfig.LegacyAPIGroupPrefixes = configprocessing.LegacyAPIGroupPrefixes
	genericConfig.RequestInfoResolver = configprocessing.OpenshiftRequestInfoResolver()
	genericConfig.OpenAPIConfig = configprocessing.DefaultOpenAPIConfig(nil)
	genericConfig.SwaggerConfig = defaultSwaggerConfig()
	genericConfig.RESTOptionsGetter = restOptsGetter
	// previously overwritten.  I don't know why
	genericConfig.RequestTimeout = time.Duration(60) * time.Second
	genericConfig.MinRequestTimeout = int(config.ServingInfo.RequestTimeoutSeconds)
	genericConfig.MaxRequestsInFlight = int(config.ServingInfo.MaxRequestsInFlight)
	genericConfig.MaxMutatingRequestsInFlight = int(config.ServingInfo.MaxRequestsInFlight / 2)
	genericConfig.LongRunningFunc = configprocessing.IsLongRunningRequest

	// I'm just hoping this works.  I don't think we use it.
	//MergedResourceConfig *serverstore.ResourceConfig

	servingOptions, err := configprocessing.ToServingOptions(config.ServingInfo)
	if err != nil {
		return nil, err
	}
	if err := servingOptions.ApplyTo(&genericConfig.Config); err != nil {
		return nil, err
	}
	authenticationOptions := genericapiserveroptions.NewDelegatingAuthenticationOptions()
	authenticationOptions.RemoteKubeConfigFile = config.KubeClientConfig.KubeConfig
	if err := authenticationOptions.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.OpenAPIConfig); err != nil {
		return nil, err
	}
	authorizationOptions := genericapiserveroptions.NewDelegatingAuthorizationOptions().WithAlwaysAllowPaths("/healthz", "/healthz/").WithAlwaysAllowGroups("system:masters")
	authorizationOptions.RemoteKubeConfigFile = config.KubeClientConfig.KubeConfig
	if err := authorizationOptions.ApplyTo(&genericConfig.Authorization); err != nil {
		return nil, err
	}

	informers, err := NewInformers(kubeInformers, kubeClientConfig, genericConfig.LoopbackClientConfig)
	if err != nil {
		return nil, err
	}
	if err := informers.GetOpenshiftUserInformers().User().V1().Groups().Informer().AddIndexers(cache.Indexers{
		usercache.ByUserIndexName: usercache.ByUserIndexKeys,
	}); err != nil {
		return nil, err
	}

	projectCache, err := NewProjectCache(informers.kubernetesInformers.Core().V1().Namespaces(), kubeClientConfig, config.ProjectConfig.DefaultNodeSelector)
	if err != nil {
		return nil, err
	}
	clusterQuotaMappingController := NewClusterQuotaMappingController(informers.kubernetesInformers.Core().V1().Namespaces(), informers.quotaInformers.Quota().InternalVersion().ClusterResourceQuotas())
	discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery())
	restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
	admissionInitializer, err := originadmission.NewPluginInitializer(config.ImagePolicyConfig.ExternalRegistryHostnames, config.ImagePolicyConfig.InternalRegistryHostname, config.CloudProviderFile, kubeClientConfig, informers, genericConfig.Authorization.Authorizer, projectCache, restMapper, clusterQuotaMappingController)
	if err != nil {
		return nil, err
	}
	namespaceLabelDecorator := namespaceconditions.NamespaceLabelConditions{
		NamespaceClient: kubeClient.CoreV1(),
		NamespaceLister: informers.GetKubernetesInformers().Core().V1().Namespaces().Lister(),

		SkipLevelZeroNames: originadmission.SkipRunLevelZeroPlugins,
		SkipLevelOneNames:  originadmission.SkipRunLevelOnePlugins,
	}
	admissionDecorators := admission.Decorators{
		admission.DecoratorFunc(namespaceLabelDecorator.WithNamespaceLabelConditions),
		admission.DecoratorFunc(admissionmetrics.WithControllerMetrics),
	}
	explicitOn := []string{}
	explicitOff := []string{}
	for plugin, config := range config.AdmissionPluginConfig {
		enabled, err := isAdmissionPluginActivated(config)
		if err != nil {
			return nil, err
		}
		if enabled {
			glog.V(2).Infof("Enabling %s", plugin)
			explicitOn = append(explicitOn, plugin)
		} else {
			glog.V(2).Infof("Disabling %s", plugin)
			explicitOff = append(explicitOff, plugin)
		}
	}
	genericConfig.AdmissionControl, err = originadmission.NewAdmissionChains([]string{}, explicitOn, explicitOff, config.AdmissionPluginConfig, admissionInitializer, admissionDecorators)
	if err != nil {
		return nil, err
	}

	var externalRegistryHostname string
	if len(config.ImagePolicyConfig.ExternalRegistryHostnames) > 0 {
		externalRegistryHostname = config.ImagePolicyConfig.ExternalRegistryHostnames[0]
	}
	registryHostnameRetriever, err := registryhostname.DefaultRegistryHostnameRetriever(kubeClientConfig, externalRegistryHostname, config.ImagePolicyConfig.InternalRegistryHostname)
	if err != nil {
		return nil, err
	}
	imageLimitVerifier := ImageLimitVerifier(informers.internalKubernetesInformers.Core().InternalVersion().LimitRanges())

	var caData []byte
	if len(config.ImagePolicyConfig.AdditionalTrustedCA) != 0 {
		glog.V(2).Infof("Image import using additional CA path: %s", config.ImagePolicyConfig.AdditionalTrustedCA)
		var err error
		caData, err = ioutil.ReadFile(config.ImagePolicyConfig.AdditionalTrustedCA)
		if err != nil {
			return nil, fmt.Errorf("failed to read CA bundle %s for image importing: %v", config.ImagePolicyConfig.AdditionalTrustedCA, err)
		}
	}

	subjectLocator := NewSubjectLocator(informers.GetKubernetesInformers().Rbac().V1())
	projectAuthorizationCache := NewProjectAuthorizationCache(
		subjectLocator,
		informers.GetInternalKubernetesInformers().Core().InternalVersion().Namespaces().Informer(),
		informers.GetKubernetesInformers().Rbac().V1(),
	)

	routeAllocator, err := configprocessing.RouteAllocator(config.RoutingConfig.Subdomain)
	if err != nil {
		return nil, err
	}

	ruleResolver := NewRuleResolver(informers.kubernetesInformers.Rbac().V1())

	ret := &OpenshiftAPIConfig{
		GenericConfig: genericConfig,
		ExtraConfig: OpenshiftAPIExtraConfig{
			InformerStart:                      informers.Start,
			KubeAPIServerClientConfig:          kubeClientConfig,
			KubeInternalInformers:              informers.internalKubernetesInformers,
			KubeInformers:                      kubeInformers, // TODO remove this and use the one from the genericconfig
			QuotaInformers:                     informers.quotaInformers,
			SecurityInformers:                  informers.securityInformers,
			RuleResolver:                       ruleResolver,
			SubjectLocator:                     subjectLocator,
			LimitVerifier:                      imageLimitVerifier,
			RegistryHostnameRetriever:          registryHostnameRetriever,
			AllowedRegistriesForImport:         config.ImagePolicyConfig.AllowedRegistriesForImport,
			MaxImagesBulkImportedPerRepository: config.ImagePolicyConfig.MaxImagesBulkImportedPerRepository,
			AdditionalTrustedCA:                caData,
			RouteAllocator:                     routeAllocator,
			ProjectAuthorizationCache:          projectAuthorizationCache,
			ProjectCache:                       projectCache,
			ProjectRequestTemplate:             config.ProjectConfig.ProjectRequestTemplate,
			ProjectRequestMessage:              config.ProjectConfig.ProjectRequestMessage,
			ClusterQuotaMappingController:      clusterQuotaMappingController,
			RESTMapper:                         restMapper,
			ServiceAccountMethod:               string(config.ServiceAccountOAuthGrantMethod),
		},
	}

	return ret, ret.ExtraConfig.Validate()
}

var apiInfo = map[string]swagger.Info{
	legacy.RESTPrefix + "/" + legacy.GroupVersion.Version: {
		Title:       "OpenShift v1 REST API",
		Description: `The OpenShift API exposes operations for managing an enterprise Kubernetes cluster, including security and user management, application deployments, image and source builds, HTTP(s) routing, and project management.`,
	},
}

// customizeSwaggerDefinition applies selective patches to the swagger API docs
// TODO: move most of these upstream or to go-restful
func customizeSwaggerDefinition(apiList *swagger.ApiDeclarationList) {
	for path, info := range apiInfo {
		if dec, ok := apiList.At(path); ok {
			if len(info.Title) > 0 {
				dec.Info.Title = info.Title
			}
			if len(info.Description) > 0 {
				dec.Info.Description = info.Description
			}
			apiList.Put(path, dec)
		} else {
			glog.Warningf("No API exists for predefined swagger description %s", path)
		}
	}
	for _, version := range []string{legacy.RESTPrefix + "/" + legacy.GroupVersion.Version} {
		apiDeclaration, _ := apiList.At(version)
		models := &apiDeclaration.Models

		model, _ := models.At("runtime.RawExtension")
		model.Required = []string{}
		model.Properties = swagger.ModelPropertyList{}
		model.Description = "this may be any JSON object with a 'kind' and 'apiVersion' field; and is preserved unmodified by processing"
		models.Put("runtime.RawExtension", model)

		model, _ = models.At("patch.Object")
		model.Description = "represents an object patch, which may be any of: JSON patch (RFC 6902), JSON merge patch (RFC 7396), or the Kubernetes strategic merge patch"
		models.Put("patch.Object", model)

		apiDeclaration.Models = *models
		apiList.Put(version, apiDeclaration)
	}
}

func defaultSwaggerConfig() *swagger.Config {
	ret := genericapiserver.DefaultSwaggerConfig()
	ret.PostBuildHandler = customizeSwaggerDefinition
	return ret
}

func OpenshiftHandlerChain(apiHandler http.Handler, genericConfig *genericapiserver.Config) http.Handler {
	// this is the normal kube handler chain
	handler := genericapiserver.DefaultBuildHandlerChain(apiHandler, genericConfig)

	handler = configprocessing.WithCacheControl(handler, "no-store") // protected endpoints should not be cached

	return handler
}

func isAdmissionPluginActivated(config configv1.AdmissionPluginConfig) (bool, error) {
	var (
		data []byte
		err  error
	)
	switch {
	case len(config.Configuration.Raw) == 0:
		data, err = ioutil.ReadFile(config.Location)
	default:
		data = config.Configuration.Raw
	}
	if err != nil {
		return false, err
	}
	return configlatest.IsAdmissionPluginActivated(bytes.NewReader(data), true)
}
<commit_msg>explicitly wire the aggregation information and use consistent client CA<commit_after>package openshiftapiserver

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"github.com/emicklei/go-restful-swagger12"
	"github.com/golang/glog"

	"k8s.io/apiserver/pkg/admission"
	admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
	genericapiserver "k8s.io/apiserver/pkg/server"
	genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
	cacheddiscovery "k8s.io/client-go/discovery/cached"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/restmapper"
	"k8s.io/client-go/tools/cache"
	"k8s.io/kubernetes/pkg/api/legacyscheme"

	configv1 "github.com/openshift/api/config/v1"
	openshiftcontrolplanev1 "github.com/openshift/api/openshiftcontrolplane/v1"
	"github.com/openshift/library-go/pkg/config/helpers"
	"github.com/openshift/origin/pkg/admission/namespaceconditions"
	"github.com/openshift/origin/pkg/api/legacy"
	originadmission "github.com/openshift/origin/pkg/apiserver/admission"
	"github.com/openshift/origin/pkg/cmd/openshift-apiserver/openshiftapiserver/configprocessing"
	configlatest "github.com/openshift/origin/pkg/cmd/server/apis/config/latest"
	"github.com/openshift/origin/pkg/image/apiserver/registryhostname"
	usercache "github.com/openshift/origin/pkg/user/cache"
	"github.com/openshift/origin/pkg/version"
)

func NewOpenshiftAPIConfig(config *openshiftcontrolplanev1.OpenShiftAPIServerConfig) (*OpenshiftAPIConfig, error) {
	kubeClientConfig, err := helpers.GetKubeClientConfig(config.KubeClientConfig)
	if err != nil {
		return nil, err
	}
	kubeClient, err := kubernetes.NewForConfig(kubeClientConfig)
	if err != nil {
		return nil, err
	}
	kubeInformers := informers.NewSharedInformerFactory(kubeClient, 10*time.Minute)

	openshiftVersion := version.Get()

	backend, policyChecker, err := configprocessing.GetAuditConfig(config.AuditConfig)
	if err != nil {
		return nil, err
	}
	restOptsGetter, err := NewRESTOptionsGetter(config.APIServerArguments, config.StorageConfig)
	if err != nil {
		return nil, err
	}

	genericConfig := genericapiserver.NewRecommendedConfig(legacyscheme.Codecs)
	// Current default values
	//Serializer:                   codecs,
	//ReadWritePort:                443,
	//BuildHandlerChainFunc:        DefaultBuildHandlerChain,
	//HandlerChainWaitGroup:        new(utilwaitgroup.SafeWaitGroup),
	//LegacyAPIGroupPrefixes:       sets.NewString(DefaultLegacyAPIPrefix),
	//DisabledPostStartHooks:       sets.NewString(),
	//HealthzChecks:                []healthz.HealthzChecker{healthz.PingHealthz, healthz.LogHealthz},
	//EnableIndex:                  true,
	//EnableDiscovery:              true,
	//EnableProfiling:              true,
	//EnableMetrics:                true,
	//MaxRequestsInFlight:          400,
	//MaxMutatingRequestsInFlight:  200,
	//RequestTimeout:               time.Duration(60) * time.Second,
	//MinRequestTimeout:            1800,
	//EnableAPIResponseCompression: utilfeature.DefaultFeatureGate.Enabled(features.APIResponseCompression),
	//LongRunningFunc: genericfilters.BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString()),

	// TODO this is actually specific to the kubeapiserver
	//RuleResolver authorizer.RuleResolver
	genericConfig.SharedInformerFactory = kubeInformers
	genericConfig.ClientConfig = kubeClientConfig

	// these are set via options
	//SecureServing *SecureServingInfo
	//Authentication AuthenticationInfo
	//Authorization AuthorizationInfo
	//LoopbackClientConfig *restclient.Config
	// this is set after the options are overlayed to get the authorizer we need.
	//AdmissionControl      admission.Interface
	//ReadWritePort int
	//PublicAddress net.IP

	// these are defaulted sanely during complete
	//DiscoveryAddresses discovery.Addresses

	genericConfig.CorsAllowedOriginList = config.CORSAllowedOrigins
	genericConfig.Version = &openshiftVersion
	// we don't use legacy audit anymore
	genericConfig.LegacyAuditWriter = nil
	genericConfig.AuditBackend = backend
	genericConfig.AuditPolicyChecker = policyChecker
	genericConfig.ExternalAddress = "apiserver.openshift-apiserver.svc"
	genericConfig.BuildHandlerChainFunc = OpenshiftHandlerChain
	genericConfig.LegacyAPIGroupPrefixes = configprocessing.LegacyAPIGroupPrefixes
	genericConfig.RequestInfoResolver = configprocessing.OpenshiftRequestInfoResolver()
	genericConfig.OpenAPIConfig = configprocessing.DefaultOpenAPIConfig(nil)
	genericConfig.SwaggerConfig = defaultSwaggerConfig()
	genericConfig.RESTOptionsGetter = restOptsGetter
	// previously overwritten.  I don't know why
	genericConfig.RequestTimeout = time.Duration(60) * time.Second
	genericConfig.MinRequestTimeout = int(config.ServingInfo.RequestTimeoutSeconds)
	genericConfig.MaxRequestsInFlight = int(config.ServingInfo.MaxRequestsInFlight)
	genericConfig.MaxMutatingRequestsInFlight = int(config.ServingInfo.MaxRequestsInFlight / 2)
	genericConfig.LongRunningFunc = configprocessing.IsLongRunningRequest

	// I'm just hoping this works.  I don't think we use it.
	//MergedResourceConfig *serverstore.ResourceConfig

	servingOptions, err := configprocessing.ToServingOptions(config.ServingInfo)
	if err != nil {
		return nil, err
	}
	if err := servingOptions.ApplyTo(&genericConfig.Config); err != nil {
		return nil, err
	}
	authenticationOptions := genericapiserveroptions.NewDelegatingAuthenticationOptions()
	// keep working for integration tests
	if len(config.AggregatorConfig.ClientCA) > 0 {
		authenticationOptions.ClientCert.ClientCA = config.ServingInfo.ClientCA
		authenticationOptions.RequestHeader.ClientCAFile = config.AggregatorConfig.ClientCA
		authenticationOptions.RequestHeader.AllowedNames = config.AggregatorConfig.AllowedNames
		authenticationOptions.RequestHeader.UsernameHeaders = config.AggregatorConfig.UsernameHeaders
		authenticationOptions.RequestHeader.GroupHeaders = config.AggregatorConfig.GroupHeaders
		authenticationOptions.RequestHeader.ExtraHeaderPrefixes = config.AggregatorConfig.ExtraHeaderPrefixes
	}
	authenticationOptions.RemoteKubeConfigFile = config.KubeClientConfig.KubeConfig
	if err := authenticationOptions.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.OpenAPIConfig); err != nil {
		return nil, err
	}
	authorizationOptions := genericapiserveroptions.NewDelegatingAuthorizationOptions().WithAlwaysAllowPaths("/healthz", "/healthz/").WithAlwaysAllowGroups("system:masters")
	authorizationOptions.RemoteKubeConfigFile = config.KubeClientConfig.KubeConfig
	if err := authorizationOptions.ApplyTo(&genericConfig.Authorization); err != nil {
		return nil, err
	}

	informers, err := NewInformers(kubeInformers, kubeClientConfig, genericConfig.LoopbackClientConfig)
	if err != nil {
		return nil, err
	}
	if err := informers.GetOpenshiftUserInformers().User().V1().Groups().Informer().AddIndexers(cache.Indexers{
		usercache.ByUserIndexName: usercache.ByUserIndexKeys,
	}); err != nil {
		return nil, err
	}

	projectCache, err := NewProjectCache(informers.kubernetesInformers.Core().V1().Namespaces(), kubeClientConfig, config.ProjectConfig.DefaultNodeSelector)
	if err != nil {
		return nil, err
	}
	clusterQuotaMappingController := NewClusterQuotaMappingController(informers.kubernetesInformers.Core().V1().Namespaces(), informers.quotaInformers.Quota().InternalVersion().ClusterResourceQuotas())
	discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery())
	restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
	admissionInitializer, err := originadmission.NewPluginInitializer(config.ImagePolicyConfig.ExternalRegistryHostnames, config.ImagePolicyConfig.InternalRegistryHostname, config.CloudProviderFile, kubeClientConfig, informers, genericConfig.Authorization.Authorizer, projectCache, restMapper, clusterQuotaMappingController)
	if err != nil {
		return nil, err
	}
	namespaceLabelDecorator := namespaceconditions.NamespaceLabelConditions{
		NamespaceClient: kubeClient.CoreV1(),
		NamespaceLister: informers.GetKubernetesInformers().Core().V1().Namespaces().Lister(),

		SkipLevelZeroNames: originadmission.SkipRunLevelZeroPlugins,
		SkipLevelOneNames:  originadmission.SkipRunLevelOnePlugins,
	}
	admissionDecorators := admission.Decorators{
		admission.DecoratorFunc(namespaceLabelDecorator.WithNamespaceLabelConditions),
		admission.DecoratorFunc(admissionmetrics.WithControllerMetrics),
	}
	explicitOn := []string{}
	explicitOff := []string{}
	for plugin, config := range config.AdmissionPluginConfig {
		enabled, err := isAdmissionPluginActivated(config)
		if err != nil {
			return nil, err
		}
		if enabled {
			glog.V(2).Infof("Enabling %s", plugin)
			explicitOn = append(explicitOn, plugin)
		} else {
			glog.V(2).Infof("Disabling %s", plugin)
			explicitOff = append(explicitOff, plugin)
		}
	}
	genericConfig.AdmissionControl, err = originadmission.NewAdmissionChains([]string{}, explicitOn, explicitOff, config.AdmissionPluginConfig, admissionInitializer, admissionDecorators)
	if err != nil {
		return nil, err
	}

	var externalRegistryHostname string
	if len(config.ImagePolicyConfig.ExternalRegistryHostnames) > 0 {
		externalRegistryHostname = config.ImagePolicyConfig.ExternalRegistryHostnames[0]
	}
	registryHostnameRetriever, err := registryhostname.DefaultRegistryHostnameRetriever(kubeClientConfig, externalRegistryHostname, config.ImagePolicyConfig.InternalRegistryHostname)
	if err != nil {
		return nil, err
	}
	imageLimitVerifier := ImageLimitVerifier(informers.internalKubernetesInformers.Core().InternalVersion().LimitRanges())

	var caData []byte
	if len(config.ImagePolicyConfig.AdditionalTrustedCA) != 0 {
		glog.V(2).Infof("Image import using additional CA path: %s", config.ImagePolicyConfig.AdditionalTrustedCA)
		var err error
		caData, err = ioutil.ReadFile(config.ImagePolicyConfig.AdditionalTrustedCA)
		if err != nil {
			return nil, fmt.Errorf("failed to read CA bundle %s for image importing: %v", config.ImagePolicyConfig.AdditionalTrustedCA, err)
		}
	}

	subjectLocator := NewSubjectLocator(informers.GetKubernetesInformers().Rbac().V1())
	projectAuthorizationCache := NewProjectAuthorizationCache(
		subjectLocator,
		informers.GetInternalKubernetesInformers().Core().InternalVersion().Namespaces().Informer(),
		informers.GetKubernetesInformers().Rbac().V1(),
	)

	routeAllocator, err := configprocessing.RouteAllocator(config.RoutingConfig.Subdomain)
	if err != nil {
		return nil, err
	}

	ruleResolver := NewRuleResolver(informers.kubernetesInformers.Rbac().V1())

	ret := &OpenshiftAPIConfig{
		GenericConfig: genericConfig,
		ExtraConfig: OpenshiftAPIExtraConfig{
			InformerStart:                      informers.Start,
			KubeAPIServerClientConfig:          kubeClientConfig,
			KubeInternalInformers:              informers.internalKubernetesInformers,
			KubeInformers:                      kubeInformers, // TODO remove this and use the one from the genericconfig
			QuotaInformers:                     informers.quotaInformers,
			SecurityInformers:                  informers.securityInformers,
			RuleResolver:                       ruleResolver,
			SubjectLocator:                     subjectLocator,
			LimitVerifier:                      imageLimitVerifier,
			RegistryHostnameRetriever:          registryHostnameRetriever,
			AllowedRegistriesForImport:         config.ImagePolicyConfig.AllowedRegistriesForImport,
			MaxImagesBulkImportedPerRepository: config.ImagePolicyConfig.MaxImagesBulkImportedPerRepository,
			AdditionalTrustedCA:                caData,
			RouteAllocator:                     routeAllocator,
			ProjectAuthorizationCache:          projectAuthorizationCache,
			ProjectCache:                       projectCache,
			ProjectRequestTemplate:             config.ProjectConfig.ProjectRequestTemplate,
			ProjectRequestMessage:              config.ProjectConfig.ProjectRequestMessage,
			ClusterQuotaMappingController:      clusterQuotaMappingController,
			RESTMapper:                         restMapper,
			ServiceAccountMethod:               string(config.ServiceAccountOAuthGrantMethod),
		},
	}

	return ret, ret.ExtraConfig.Validate()
}

var apiInfo = map[string]swagger.Info{
	legacy.RESTPrefix + "/" + legacy.GroupVersion.Version: {
		Title:       "OpenShift v1 REST API",
		Description: `The OpenShift API exposes operations for managing an enterprise Kubernetes cluster, including security and user management, application deployments, image and source builds, HTTP(s) routing, and project management.`,
	},
}

// customizeSwaggerDefinition applies selective patches to the swagger API docs
// TODO: move most of these upstream or to go-restful
func customizeSwaggerDefinition(apiList *swagger.ApiDeclarationList) {
	for path, info := range apiInfo {
		if dec, ok := apiList.At(path); ok {
			if len(info.Title) > 0 {
				dec.Info.Title = info.Title
			}
			if len(info.Description) > 0 {
				dec.Info.Description = info.Description
			}
			apiList.Put(path, dec)
		} else {
			glog.Warningf("No API exists for predefined swagger description %s", path)
		}
	}
	for _, version := range []string{legacy.RESTPrefix + "/" + legacy.GroupVersion.Version} {
		apiDeclaration, _ := apiList.At(version)
		models := &apiDeclaration.Models

		model, _ := models.At("runtime.RawExtension")
		model.Required = []string{}
		model.Properties = swagger.ModelPropertyList{}
		model.Description = "this may be any JSON object with a 'kind' and 'apiVersion' field; and is preserved unmodified by processing"
		models.Put("runtime.RawExtension", model)

		model, _ = models.At("patch.Object")
		model.Description = "represents an object patch, which may be any of: JSON patch (RFC 6902), JSON merge patch (RFC 7396), or the Kubernetes strategic merge patch"
		models.Put("patch.Object", model)

		apiDeclaration.Models = *models
		apiList.Put(version, apiDeclaration)
	}
}

func defaultSwaggerConfig() *swagger.Config {
	ret := genericapiserver.DefaultSwaggerConfig()
	ret.PostBuildHandler = customizeSwaggerDefinition
	return ret
}

func OpenshiftHandlerChain(apiHandler http.Handler, genericConfig *genericapiserver.Config) http.Handler {
	// this is the normal kube handler chain
	handler := genericapiserver.DefaultBuildHandlerChain(apiHandler, genericConfig)

	handler = configprocessing.WithCacheControl(handler, "no-store") // protected endpoints should not be cached

	return handler
}

func isAdmissionPluginActivated(config configv1.AdmissionPluginConfig) (bool, error) {
	var (
		data []byte
		err  error
	)
	switch {
	case len(config.Configuration.Raw) == 0:
		data, err = ioutil.ReadFile(config.Location)
	default:
		data = config.Configuration.Raw
	}
	if err != nil {
		return false, err
	}
	return configlatest.IsAdmissionPluginActivated(bytes.NewReader(data), true)
}
<|endoftext|>


<commit_before>// Tideland Go REST Server Library - JSON Web Token - Header
//
// Copyright (C) 2016 Frank Mueller / Tideland / Oldenburg / Germany
//
// All rights reserved. Use of this source code is governed
// by the new BSD license.

package jwt

//--------------------
// IMPORTS
//--------------------

import (
	"net/http"
	"strings"

	"github.com/tideland/gorest/rest"
)

//--------------------
// REQUEST AND JOB HANDLING
//--------------------

// AddToRequest adds a token as header to a request for
// usage by a client.
func AddToRequest(req *http.Request, jwt JWT) *http.Request {
	req.Header.Add("Authorization", "Bearer "+jwt.String())
	return req
}

// DecodeFromRequest tries to retrieve a token from a request
// header.
func DecodeFromRequest(req *http.Request) (JWT, error) {
	return nil, nil
}

// DecodeFromJob retrieves a possible JWT from
// the request inside a REST job. The JWT is only decoded.
func DecodeFromJob(job rest.Job) (JWT, error) {
	return retrieveFromJob(job, nil, nil)
}

// DecodeCachedFromJob retrieves a possible JWT from the request
// inside a REST job and checks if it already is cached. The JWT is
// only decoded. In case of no error the token is added to the cache.
func DecodeCachedFromJob(job rest.Job, cache Cache) (JWT, error) {
	return retrieveFromJob(job, cache, nil)
}

// VerifyFromJob retrieves a possible JWT from
// the request inside a REST job. The JWT is verified.
func VerifyFromJob(job rest.Job, key Key) (JWT, error) {
	return retrieveFromJob(job, nil, key)
}

// VerifyCachedFromJob retrieves a possible JWT from the request
// inside a REST job and checks if it already is cached. The JWT is
// verified. In case of no error the token is added to the cache.
func VerifyCachedFromJob(job rest.Job, cache Cache, key Key) (JWT, error) {
	return retrieveFromJob(job, cache, key)
}

//--------------------
// PRIVATE HELPERS
//--------------------

// retrieveFromRequest is the generic retrieval function with possible
// caching and verification.
func retrieveFromRequest(req *http.Request, cache Cache, key Key) (JWT, error) {
	// Retrieve token from header.
	authorization := req.Header.Get("Authorization")
	if authorization == "" {
		// TODO(mue): Add error.
		return nil, nil
	}
	fields := strings.Fields(authorization)
	if len(fields) != 2 || fields[0] != "Bearer" {
		// TODO(mue): Add error.
		return nil, nil
	}
	// Check cache.
	if cache != nil {
		jwt, ok := cache.Get(fields[1])
		if ok {
			return jwt, nil
		}
	}
	// Decode or verify.
	var jwt JWT
	var err error
	if key == nil {
		jwt, err = Decode(fields[1])
	} else {
		jwt, err = Verify(fields[1], key)
	}
	if err != nil {
		return nil, err
	}
	// Add to cache and return.
	if cache != nil {
		cache.Put(jwt)
	}
	return jwt, nil
}

// EOF
<commit_msg>Made compiling again<commit_after>// Tideland Go REST Server Library - JSON Web Token - Header
//
// Copyright (C) 2016 Frank Mueller / Tideland / Oldenburg / Germany
//
// All rights reserved. Use of this source code is governed
// by the new BSD license.

package jwt

//--------------------
// IMPORTS
//--------------------

import (
	"net/http"
	"strings"

	"github.com/tideland/gorest/rest"
)

//--------------------
// REQUEST AND JOB HANDLING
//--------------------

// AddToRequest adds a token as header to a request for
// usage by a client.
func AddToRequest(req *http.Request, jwt JWT) *http.Request {
	req.Header.Add("Authorization", "Bearer "+jwt.String())
	return req
}

// DecodeFromRequest tries to retrieve a token from a request
// header.
func DecodeFromRequest(req *http.Request) (JWT, error) {
	return nil, nil
}

// DecodeFromJob retrieves a possible JWT from
// the request inside a REST job. The JWT is only decoded.
func DecodeFromJob(job rest.Job) (JWT, error) {
	return retrieveFromRequest(job.Request(), nil, nil)
}

// DecodeCachedFromJob retrieves a possible JWT from the request
// inside a REST job and checks if it already is cached. The JWT is
// only decoded. In case of no error the token is added to the cache.
func DecodeCachedFromJob(job rest.Job, cache Cache) (JWT, error) {
	return retrieveFromRequest(job.Request(), cache, nil)
}

// VerifyFromJob retrieves a possible JWT from
// the request inside a REST job. The JWT is verified.
func VerifyFromJob(job rest.Job, key Key) (JWT, error) {
	return retrieveFromRequest(job.Request(), nil, key)
}

// VerifyCachedFromJob retrieves a possible JWT from the request
// inside a REST job and checks if it already is cached. The JWT is
// verified. In case of no error the token is added to the cache.
func VerifyCachedFromJob(job rest.Job, cache Cache, key Key) (JWT, error) {
	return retrieveFromRequest(job.Request(), cache, key)
}

//--------------------
// PRIVATE HELPERS
//--------------------

// retrieveFromRequest is the generic retrieval function with possible
// caching and verification.
func retrieveFromRequest(req *http.Request, cache Cache, key Key) (JWT, error) {
	// Retrieve token from header.
	authorization := req.Header.Get("Authorization")
	if authorization == "" {
		// TODO(mue): Add error.
		return nil, nil
	}
	fields := strings.Fields(authorization)
	if len(fields) != 2 || fields[0] != "Bearer" {
		// TODO(mue): Add error.
		return nil, nil
	}
	// Check cache.
	if cache != nil {
		jwt, ok := cache.Get(fields[1])
		if ok {
			return jwt, nil
		}
	}
	// Decode or verify.
	var jwt JWT
	var err error
	if key == nil {
		jwt, err = Decode(fields[1])
	} else {
		jwt, err = Verify(fields[1], key)
	}
	if err != nil {
		return nil, err
	}
	// Add to cache and return.
	if cache != nil {
		cache.Put(jwt)
	}
	return jwt, nil
}

// EOF
<|endoftext|>


<commit_before>package helpers

import (
	"context"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"sigs.k8s.io/yaml"

	"github.com/werf/logboek"
	"github.com/werf/werf/pkg/image"
	"github.com/werf/werf/pkg/werf"
)

func NewChartExtenderServiceValuesData() *ChartExtenderServiceValuesData {
	return &ChartExtenderServiceValuesData{ServiceValues: make(map[string]interface{})}
}

type ChartExtenderServiceValuesData struct {
	ServiceValues map[string]interface{}
}

func (d *ChartExtenderServiceValuesData) GetServiceValues() map[string]interface{} {
	return d.ServiceValues
}

func (d *ChartExtenderServiceValuesData) SetServiceValues(vals map[string]interface{}) {
	d.ServiceValues = vals
}

type ServiceValuesOptions struct {
	Namespace       string
	Env             string
	IsStub          bool
	StubImagesNames []string

	SetDockerConfigJsonValue bool
	DockerConfigPath         string
}

func GetServiceValues(ctx context.Context, projectName string, repo string, imageInfoGetters []*image.InfoGetter, opts ServiceValuesOptions) (map[string]interface{}, error) {
	globalInfo := map[string]interface{}{
		"werf": map[string]interface{}{
			"name":    projectName,
			"version": werf.Version,
		},
	}

	werfInfo := map[string]interface{}{
		"name":    projectName,
		"version": werf.Version,
		"repo":    repo,
		"image":   map[string]interface{}{},
	}

	if opts.Env != "" {
		globalInfo["env"] = opts.Env
		werfInfo["env"] = opts.Env
	} else if opts.IsStub {
		globalInfo["env"] = ""
		werfInfo["env"] = ""
	}

	if opts.Namespace != "" {
		werfInfo["namespace"] = opts.Namespace
	}

	if opts.IsStub {
		stubImage := fmt.Sprintf("%s:TAG", repo)

		werfInfo["is_stub"] = true
		werfInfo["stub_image"] = stubImage
		for _, name := range opts.StubImagesNames {
			werfInfo["image"].(map[string]interface{})[name] = stubImage
		}
	}

	for _, imageInfoGetter := range imageInfoGetters {
		if imageInfoGetter.IsNameless() {
			werfInfo["is_nameless_image"] = true
			werfInfo["nameless_image"] = imageInfoGetter.GetName()
		} else {
			werfInfo["image"].(map[string]interface{})[imageInfoGetter.GetWerfImageName()] = imageInfoGetter.GetName()
		}
	}

	res := map[string]interface{}{
		"werf":   werfInfo,
		"global": globalInfo,
	}

	if opts.SetDockerConfigJsonValue {
		if err := writeDockerConfigJsonValue(ctx, res, opts.DockerConfigPath); err != nil {
			return nil, fmt.Errorf("error writing docker config value: %s", err)
		}
	}

	data, err := yaml.Marshal(res)
	logboek.Context(ctx).Debug().LogF("GetServiceValues result (err=%s):\n%s\n", err, data)

	return res, nil
}

func GetBundleServiceValues(ctx context.Context, opts ServiceValuesOptions) (map[string]interface{}, error) {
	globalInfo := map[string]interface{}{
		"werf": map[string]interface{}{
			"version": werf.Version,
		},
	}

	werfInfo := map[string]interface{}{
		"version": werf.Version,
	}

	if opts.Env != "" {
		globalInfo["env"] = opts.Env
		werfInfo["env"] = opts.Env
	}

	if opts.Namespace != "" {
		werfInfo["namespace"] = opts.Namespace
	}

	res := map[string]interface{}{
		"werf":   werfInfo,
		"global": globalInfo,
	}

	if opts.SetDockerConfigJsonValue {
		if err := writeDockerConfigJsonValue(ctx, res, opts.DockerConfigPath); err != nil {
			return nil, fmt.Errorf("error writing docker config value: %s", err)
		}
	}

	data, err := yaml.Marshal(res)
	logboek.Context(ctx).Debug().LogF("GetBundleServiceValues result (err=%s):\n%s\n", err, data)

	return res, nil
}

func writeDockerConfigJsonValue(ctx context.Context, values map[string]interface{}, dockerConfigPath string) error {
	if dockerConfigPath == "" {
		dockerConfigPath = filepath.Join(os.Getenv("HOME"), ".docker")
	}
	configJsonPath := filepath.Join(dockerConfigPath, "config.json")

	if _, err := os.Stat(configJsonPath); os.IsNotExist(err) {
		return nil
	} else if err != nil {
		return fmt.Errorf("error accessing %q: %s", configJsonPath, err)
	}

	if data, err := ioutil.ReadFile(configJsonPath); err != nil {
		return fmt.Errorf("error reading %q: %s", configJsonPath, err)
	} else {
		values["dockerconfigjson"] = base64.StdEncoding.EncodeToString(data)
	}

	logboek.Context(ctx).Default().LogF("NOTE: ### --set-docker-config-json-value option has been specified ###\n")
	logboek.Context(ctx).Default().LogF("NOTE:\n")
	logboek.Context(ctx).Default().LogF("NOTE: Werf sets .Values.dockerconfigjson with the current docker config content %q with --set-docker-config-json-value option.\n", configJsonPath)
	logboek.Context(ctx).Default().LogF("NOTE: This docker config may contain temporal login credentials created using temporal short-lived token (CI_JOB_TOKEN for example),\n")
	logboek.Context(ctx).Default().LogF("NOTE: and in such case should not be used as imagePullSecrets.\n")

	return nil
}
<commit_msg>feat: expose "tag" service value<commit_after>package helpers

import (
	"context"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"sigs.k8s.io/yaml"

	"github.com/werf/logboek"
	"github.com/werf/werf/pkg/image"
	"github.com/werf/werf/pkg/werf"
)

func NewChartExtenderServiceValuesData() *ChartExtenderServiceValuesData {
	return &ChartExtenderServiceValuesData{ServiceValues: make(map[string]interface{})}
}

type ChartExtenderServiceValuesData struct {
	ServiceValues map[string]interface{}
}

func (d *ChartExtenderServiceValuesData) GetServiceValues() map[string]interface{} {
	return d.ServiceValues
}

func (d *ChartExtenderServiceValuesData) SetServiceValues(vals map[string]interface{}) {
	d.ServiceValues = vals
}

type ServiceValuesOptions struct {
	Namespace       string
	Env             string
	IsStub          bool
	StubImagesNames []string

	SetDockerConfigJsonValue bool
	DockerConfigPath         string
}

func GetServiceValues(ctx context.Context, projectName string, repo string, imageInfoGetters []*image.InfoGetter, opts ServiceValuesOptions) (map[string]interface{}, error) {
	globalInfo := map[string]interface{}{
		"werf": map[string]interface{}{
			"name":    projectName,
			"version": werf.Version,
		},
	}

	werfInfo := map[string]interface{}{
		"name":    projectName,
		"version": werf.Version,
		"repo":    repo,
		"image":   map[string]interface{}{},
		"tag":     map[string]interface{}{},
	}

	if opts.Env != "" {
		globalInfo["env"] = opts.Env
		werfInfo["env"] = opts.Env
	} else if opts.IsStub {
		globalInfo["env"] = ""
		werfInfo["env"] = ""
	}

	if opts.Namespace != "" {
		werfInfo["namespace"] = opts.Namespace
	}

	if opts.IsStub {
		stubTag := "TAG"
		stubImage := fmt.Sprintf("%s:%s", repo, stubTag)

		werfInfo["is_stub"] = true
		werfInfo["stub_image"] = stubImage
		for _, name := range opts.StubImagesNames {
			werfInfo["image"].(map[string]interface{})[name] = stubImage
			werfInfo["tag"].(map[string]interface{})[name] = stubTag
		}
	}

	for _, imageInfoGetter := range imageInfoGetters {
		if imageInfoGetter.IsNameless() {
			werfInfo["is_nameless_image"] = true
			werfInfo["nameless_image"] = imageInfoGetter.GetName()
		} else {
			werfInfo["image"].(map[string]interface{})[imageInfoGetter.GetWerfImageName()] = imageInfoGetter.GetName()
			werfInfo["tag"].(map[string]interface{})[imageInfoGetter.GetWerfImageName()] = imageInfoGetter.GetTag()
		}
	}

	res := map[string]interface{}{
		"werf":   werfInfo,
		"global": globalInfo,
	}

	if opts.SetDockerConfigJsonValue {
		if err := writeDockerConfigJsonValue(ctx, res, opts.DockerConfigPath); err != nil {
			return nil, fmt.Errorf("error writing docker config value: %s", err)
		}
	}

	data, err := yaml.Marshal(res)
	logboek.Context(ctx).Debug().LogF("GetServiceValues result (err=%s):\n%s\n", err, data)

	return res, nil
}

func GetBundleServiceValues(ctx context.Context, opts ServiceValuesOptions) (map[string]interface{}, error) {
	globalInfo := map[string]interface{}{
		"werf": map[string]interface{}{
			"version": werf.Version,
		},
	}

	werfInfo := map[string]interface{}{
		"version": werf.Version,
	}

	if opts.Env != "" {
		globalInfo["env"] = opts.Env
		werfInfo["env"] = opts.Env
	}

	if opts.Namespace != "" {
		werfInfo["namespace"] = opts.Namespace
	}

	res := map[string]interface{}{
		"werf":   werfInfo,
		"global": globalInfo,
	}

	if opts.SetDockerConfigJsonValue {
		if err := writeDockerConfigJsonValue(ctx, res, opts.DockerConfigPath); err != nil {
			return nil, fmt.Errorf("error writing docker config value: %s", err)
		}
	}

	data, err := yaml.Marshal(res)
	logboek.Context(ctx).Debug().LogF("GetBundleServiceValues result (err=%s):\n%s\n", err, data)

	return res, nil
}

func writeDockerConfigJsonValue(ctx context.Context, values map[string]interface{}, dockerConfigPath string) error {
	if dockerConfigPath == "" {
		dockerConfigPath = filepath.Join(os.Getenv("HOME"), ".docker")
	}
	configJsonPath := filepath.Join(dockerConfigPath, "config.json")

	if _, err := os.Stat(configJsonPath); os.IsNotExist(err) {
		return nil
	} else if err != nil {
		return fmt.Errorf("error accessing %q: %s", configJsonPath, err)
	}

	if data, err := ioutil.ReadFile(configJsonPath); err != nil {
		return fmt.Errorf("error reading %q: %s", configJsonPath, err)
	} else {
		values["dockerconfigjson"] = base64.StdEncoding.EncodeToString(data)
	}

	logboek.Context(ctx).Default().LogF("NOTE: ### --set-docker-config-json-value option has been specified ###\n")
	logboek.Context(ctx).Default().LogF("NOTE:\n")
	logboek.Context(ctx).Default().LogF("NOTE: Werf sets .Values.dockerconfigjson with the current docker config content %q with --set-docker-config-json-value option.\n", configJsonPath)
	logboek.Context(ctx).Default().LogF("NOTE: This docker config may contain temporal login credentials created using temporal short-lived token (CI_JOB_TOKEN for example),\n")
	logboek.Context(ctx).Default().LogF("NOTE: and in such case should not be used as imagePullSecrets.\n")

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package prometheus

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/klog"
	"k8s.io/perf-tests/clusterloader2/pkg/config"
	"k8s.io/perf-tests/clusterloader2/pkg/framework"
	"k8s.io/perf-tests/clusterloader2/pkg/framework/client"
)

const (
	namespace                    = "monitoring"
	checkPrometheusReadyInterval = 30 * time.Second
	checkPrometheusReadyTimeout  = 10 * time.Minute
)

// SetUpPrometheusStack sets up prometheus stack in the cluster.
// This method is idempotent, if the prometheus stack is already set up applying the manifests
// again will be no-op.
func SetUpPrometheusStack(
	framework *framework.Framework, clusterLoaderConfig *config.ClusterLoaderConfig) error {

	k8sClient := framework.GetClientSets().GetClient()
	nodeCount := clusterLoaderConfig.ClusterConfig.Nodes

	klog.Info("Setting up prometheus stack")
	if err := client.CreateNamespace(k8sClient, namespace); err != nil {
		return err
	}
	if err := applyManifests(framework, clusterLoaderConfig); err != nil {
		return err
	}
	if err := waitForPrometheusToBeHealthy(k8sClient, nodeCount); err != nil {
		return err
	}
	klog.Info("Prometheus stack set up successfully")
	return nil
}

// TearDownPrometheusStack tears down prometheus stack, releasing all prometheus resources.
func TearDownPrometheusStack(framework *framework.Framework) error {
	klog.Info("Tearing down prometheus stack")
	k8sClient := framework.GetClientSets().GetClient()
	if err := client.DeleteNamespace(k8sClient, namespace); err != nil {
		return err
	}
	if err := client.WaitForDeleteNamespace(k8sClient, namespace); err != nil {
		return err
	}
	return nil
}

func applyManifests(
	framework *framework.Framework, clusterLoaderConfig *config.ClusterLoaderConfig) error {
	// TODO(mm4tt): Consider using the out-of-the-box "kubectl create -f".
	manifestGlob := os.ExpandEnv(
		"$GOPATH/src/k8s.io/perf-tests/clusterloader2/pkg/prometheus/manifests/*.yaml")
	templateProvider := config.NewTemplateProvider(filepath.Dir(manifestGlob))
	mapping, errList := config.GetMapping(clusterLoaderConfig)
	if errList != nil && !errList.IsEmpty() {
		return errList
	}
	manifests, err := filepath.Glob(manifestGlob)
	if err != nil {
		return err
	}
	for _, manifest := range manifests {
		klog.Infof("Applying %s\n", manifest)
		obj, err := templateProvider.TemplateToObject(filepath.Base(manifest), mapping)
		if err != nil {
			return err
		}
		if obj.IsList() {
			objList, err := obj.ToList()
			if err != nil {
				return err
			}
			for _, item := range objList.Items {
				if err := framework.CreateObject(item.GetNamespace(), item.GetName(), &item); err != nil {
					return fmt.Errorf("error while applying (%s): %v", manifest, err)
				}
			}
		} else {
			if err := framework.CreateObject(obj.GetNamespace(), obj.GetName(), obj); err != nil {
				return fmt.Errorf("error while applying (%s): %v", manifest, err)
			}
		}
	}
	return nil
}

func waitForPrometheusToBeHealthy(client clientset.Interface, nodeCount int) error {
	klog.Info("Waiting for Prometheus stack to become healthy...")
	return wait.Poll(
		checkPrometheusReadyInterval,
		checkPrometheusReadyTimeout,
		func() (bool, error) { return isPrometheusReady(client, nodeCount) })
}

func isPrometheusReady(client clientset.Interface, nodeCount int) (bool, error) {
	raw, err := client.CoreV1().
		Services(namespace).
		ProxyGet("http", "prometheus-k8s", "9090", "api/v1/targets", nil /*params*/).
		DoRaw()
	if err != nil {
		// This might happen if prometheus server is temporary down, log error but don't return it.
		klog.Warningf("error while calling prometheus api: %v", err)
		return false, nil
	}

	var response targetsResponse
	if err := json.Unmarshal(raw, &response); err != nil {
		// This shouldn't happen, return error.
		return false, err
	}

	if len(response.Data.ActiveTargets) < nodeCount {
		// There should be at least as many targets as number of nodes (e.g. there is a kube-proxy
		// instance on each node). This is a safeguard from a race condition where the prometheus
		// server is started before targets are registered.
		klog.Infof("Less active targets (%d) than nodes (%d), waiting for more to become active...",
			len(response.Data.ActiveTargets), nodeCount)
		return false, nil
	}

	for _, t := range response.Data.ActiveTargets {
		if t.Health != "up" {
			klog.Infof("Target {job=%s, pod=%s} not ready...", t.Labels["job"], t.Labels["pod"])
			return false, nil
		}
	}
	klog.Infof("All %d targets are ready", len(response.Data.ActiveTargets))
	return true, nil
}

type targetsResponse struct {
	Data targetsData `json:"data""`
}

type targetsData struct {
	ActiveTargets []target `json:"activeTargets"`
}

type target struct {
	Labels map[string]string `json:"labels"`
	Health string            `json:"health"`
}
<commit_msg>Change how CL2 waits for prometheus to be ready<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package prometheus

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/klog"
	"k8s.io/perf-tests/clusterloader2/pkg/config"
	"k8s.io/perf-tests/clusterloader2/pkg/framework"
	"k8s.io/perf-tests/clusterloader2/pkg/framework/client"
)

const (
	namespace                    = "monitoring"
	checkPrometheusReadyInterval = 30 * time.Second
	checkPrometheusReadyTimeout  = 15 * time.Minute
)

// SetUpPrometheusStack sets up prometheus stack in the cluster.
// This method is idempotent, if the prometheus stack is already set up applying the manifests
// again will be no-op.
func SetUpPrometheusStack(
	framework *framework.Framework, clusterLoaderConfig *config.ClusterLoaderConfig) error {

	k8sClient := framework.GetClientSets().GetClient()
	nodeCount := clusterLoaderConfig.ClusterConfig.Nodes

	klog.Info("Setting up prometheus stack")
	if err := client.CreateNamespace(k8sClient, namespace); err != nil {
		return err
	}
	if err := applyManifests(framework, clusterLoaderConfig); err != nil {
		return err
	}
	if err := waitForPrometheusToBeHealthy(k8sClient, nodeCount); err != nil {
		return err
	}
	klog.Info("Prometheus stack set up successfully")
	return nil
}

// TearDownPrometheusStack tears down prometheus stack, releasing all prometheus resources.
func TearDownPrometheusStack(framework *framework.Framework) error {
	klog.Info("Tearing down prometheus stack")
	k8sClient := framework.GetClientSets().GetClient()
	if err := client.DeleteNamespace(k8sClient, namespace); err != nil {
		return err
	}
	if err := client.WaitForDeleteNamespace(k8sClient, namespace); err != nil {
		return err
	}
	return nil
}

func applyManifests(
	framework *framework.Framework, clusterLoaderConfig *config.ClusterLoaderConfig) error {
	// TODO(mm4tt): Consider using the out-of-the-box "kubectl create -f".
	manifestGlob := os.ExpandEnv(
		"$GOPATH/src/k8s.io/perf-tests/clusterloader2/pkg/prometheus/manifests/*.yaml")
	templateProvider := config.NewTemplateProvider(filepath.Dir(manifestGlob))
	mapping, errList := config.GetMapping(clusterLoaderConfig)
	if errList != nil && !errList.IsEmpty() {
		return errList
	}
	manifests, err := filepath.Glob(manifestGlob)
	if err != nil {
		return err
	}
	for _, manifest := range manifests {
		klog.Infof("Applying %s\n", manifest)
		obj, err := templateProvider.TemplateToObject(filepath.Base(manifest), mapping)
		if err != nil {
			return err
		}
		if obj.IsList() {
			objList, err := obj.ToList()
			if err != nil {
				return err
			}
			for _, item := range objList.Items {
				if err := framework.CreateObject(item.GetNamespace(), item.GetName(), &item); err != nil {
					return fmt.Errorf("error while applying (%s): %v", manifest, err)
				}
			}
		} else {
			if err := framework.CreateObject(obj.GetNamespace(), obj.GetName(), obj); err != nil {
				return fmt.Errorf("error while applying (%s): %v", manifest, err)
			}
		}
	}
	return nil
}

func waitForPrometheusToBeHealthy(client clientset.Interface, nodeCount int) error {
	klog.Info("Waiting for Prometheus stack to become healthy...")
	return wait.Poll(
		checkPrometheusReadyInterval,
		checkPrometheusReadyTimeout,
		func() (bool, error) { return isPrometheusReady(client, nodeCount) })
}

func isPrometheusReady(client clientset.Interface, nodeCount int) (bool, error) {
	raw, err := client.CoreV1().
		Services(namespace).
		ProxyGet("http", "prometheus-k8s", "9090", "api/v1/targets", nil /*params*/).
		DoRaw()
	if err != nil {
		// This might happen if prometheus server is temporary down, log error but don't return it.
		klog.Warningf("error while calling prometheus api: %v", err)
		return false, nil
	}

	var response targetsResponse
	if err := json.Unmarshal(raw, &response); err != nil {
		// This shouldn't happen, return error.
		return false, err
	}

	if len(response.Data.ActiveTargets) < nodeCount {
		// There should be at least as many targets as number of nodes (e.g. there is a kube-proxy
		// instance on each node). This is a safeguard from a race condition where the prometheus
		// server is started before targets are registered.
		klog.Infof("Less active targets (%d) than nodes (%d), waiting for more to become active...",
			len(response.Data.ActiveTargets), nodeCount)
		return false, nil
	}

	nReady := 0
	for _, t := range response.Data.ActiveTargets {
		if t.Health == "up" {
			nReady++
		}
	}
	if nReady < len(response.Data.ActiveTargets) {
		klog.Infof("%d/%d targets are ready", nReady, len(response.Data.ActiveTargets))
		return false, nil
	}
	klog.Infof("All %d targets are ready", len(response.Data.ActiveTargets))
	return true, nil
}

type targetsResponse struct {
	Data targetsData `json:"data""`
}

type targetsData struct {
	ActiveTargets []target `json:"activeTargets"`
}

type target struct {
	Labels map[string]string `json:"labels"`
	Health string            `json:"health"`
}
<|endoftext|>


<commit_before>// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package model

import (
	"sort"
	"time"

	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/db"
	"github.com/syncthing/syncthing/lib/events"
	"github.com/syncthing/syncthing/lib/ignore"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/util"
	"github.com/syncthing/syncthing/lib/versioner"
)

func init() {
	folderFactories[config.FolderTypeReceiveOnly] = newReceiveOnlyFolder
}

/*
receiveOnlyFolder is a folder that does not propagate local changes outward.
It does this by the following general mechanism (not all of which is
implemented in this file):

- Local changes are scanned and versioned as usual, but get the
  FlagLocalReceiveOnly bit set.

- When changes are sent to the cluster this bit gets converted to the
  Invalid bit (like all other local flags, currently) and also the Version
  gets set to the empty version. The reason for clearing the Version is to
  ensure that other devices will not consider themselves out of date due to
  our change.

- The database layer accounts sizes per flag bit, so we can know how many
  files have been changed locally. We use this to trigger a "Revert" option
  on the folder when the amount of locally changed data is nonzero.

- To revert we take the files which have changed and reset their version
  counter down to zero. The next pull will replace our changed version with
  the globally latest. As this is a user-initiated operation we do not cause
  conflict copies when reverting.

- When pulling normally (i.e., not in the revert case) with local changes,
  normal conflict resolution will apply. Conflict copies will be created,
  but not propagated outwards (because receive only, right).

Implementation wise a receiveOnlyFolder is just a sendReceiveFolder that
sets an extra bit on local changes and has a Revert method.
*/
type receiveOnlyFolder struct {
	*sendReceiveFolder
}

func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service {
	sr := newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)
	sr.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files
	return &receiveOnlyFolder{sr}
}

func (f *receiveOnlyFolder) Revert() {
	f.doInSync(f.revert)
}

func (f *receiveOnlyFolder) revert() error {
	l.Infof("Reverting folder %v", f.Description)

	f.setState(FolderScanning)
	defer f.setState(FolderIdle)

	scanChan := make(chan string)
	go f.pullScannerRoutine(scanChan)
	defer close(scanChan)

	delQueue := &deleteQueue{
		handler:  f, // for the deleteItemOnDisk and deleteDirOnDisk methods
		ignores:  f.ignores,
		scanChan: scanChan,
	}

	batch := db.NewFileInfoBatch(func(files []protocol.FileInfo) error {
		f.updateLocalsFromScanning(files)
		return nil
	})
	snap, err := f.dbSnapshot()
	if err != nil {
		return err
	}
	defer snap.Release()
	snap.WithHave(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
		fi := intf.(protocol.FileInfo)
		if !fi.IsReceiveOnlyChanged() {
			// We're only interested in files that have changed locally in
			// receive only mode.
			return true
		}

		fi.LocalFlags &^= protocol.FlagLocalReceiveOnly

		switch gf, ok := snap.GetGlobal(fi.Name); {
		case !ok:
			msg := "Unexpected global file that we have locally"
			l.Debugf("%v revert: %v: %v", f, msg, fi.Name)
			f.evLogger.Log(events.Failure, msg)
			return true
		case gf.IsReceiveOnlyChanged():
			// The global file is our own. A revert then means to delete it.
			// We'll delete files directly, directories get queued and
			// handled below.
			if fi.Deleted {
				fi.Version = protocol.Vector{} // if this file ever resurfaces anywhere we want our delete to be strictly older
				break
			}
			handled, err := delQueue.handle(fi, snap)
			if err != nil {
				l.Infof("Revert: deleting %s: %v\n", fi.Name, err)
				return true // continue
			}
			if !handled {
				return true // continue
			}
			fi.SetDeleted(f.shortID)
			fi.Version = protocol.Vector{} // if this file ever resurfaces anywhere we want our delete to be strictly older
		case gf.IsEquivalentOptional(fi, f.modTimeWindow, false, false, protocol.FlagLocalReceiveOnly):
			// What we have locally is equivalent to the global file.
			fi = gf
		default:
			// Revert means to throw away our local changes. We reset the
			// version to the empty vector, which is strictly older than any
			// other existing version. It is not in conflict with anything,
			// either, so we will not create a conflict copy of our local
			// changes.
			fi.Version = protocol.Vector{}
		}

		batch.Append(fi)
		_ = batch.FlushIfFull()

		return true
	})
	_ = batch.Flush()

	// Handle any queued directories
	deleted, err := delQueue.flush(snap)
	if err != nil {
		l.Infoln("Revert:", err)
	}
	now := time.Now()
	for _, dir := range deleted {
		batch.Append(protocol.FileInfo{
			Name:       dir,
			Type:       protocol.FileInfoTypeDirectory,
			ModifiedS:  now.Unix(),
			ModifiedBy: f.shortID,
			Deleted:    true,
			Version:    protocol.Vector{},
		})
	}
	_ = batch.Flush()

	// We will likely have changed our local index, but that won't trigger a
	// pull by itself. Make sure we schedule one so that we start
	// downloading files.
	f.SchedulePull()

	return nil
}

// deleteQueue handles deletes by delegating to a handler and queuing
// directories for last.
type deleteQueue struct {
	handler interface {
		deleteItemOnDisk(item protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) error
		deleteDirOnDisk(dir string, snap *db.Snapshot, scanChan chan<- string) error
	}
	ignores  *ignore.Matcher
	dirs     []string
	scanChan chan<- string
}

func (q *deleteQueue) handle(fi protocol.FileInfo, snap *db.Snapshot) (bool, error) {
	// Things that are ignored but not marked deletable are not processed.
	ign := q.ignores.Match(fi.Name)
	if ign.IsIgnored() && !ign.IsDeletable() {
		return false, nil
	}

	// Directories are queued for later processing.
	if fi.IsDirectory() {
		q.dirs = append(q.dirs, fi.Name)
		return false, nil
	}

	// Kill it.
	err := q.handler.deleteItemOnDisk(fi, snap, q.scanChan)
	return true, err
}

func (q *deleteQueue) flush(snap *db.Snapshot) ([]string, error) {
	// Process directories from the leaves inward.
	sort.Sort(sort.Reverse(sort.StringSlice(q.dirs)))

	var firstError error
	var deleted []string

	for _, dir := range q.dirs {
		if err := q.handler.deleteDirOnDisk(dir, snap, q.scanChan); err == nil {
			deleted = append(deleted, dir)
		} else if err != nil && firstError == nil {
			firstError = err
		}
	}

	return deleted, firstError
}
<commit_msg>lib/model: Correct "reverting folder" log entry<commit_after>// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package model

import (
	"sort"
	"time"

	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/db"
	"github.com/syncthing/syncthing/lib/events"
	"github.com/syncthing/syncthing/lib/ignore"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/util"
	"github.com/syncthing/syncthing/lib/versioner"
)

func init() {
	folderFactories[config.FolderTypeReceiveOnly] = newReceiveOnlyFolder
}

/*
receiveOnlyFolder is a folder that does not propagate local changes outward.
It does this by the following general mechanism (not all of which is
implemented in this file):

- Local changes are scanned and versioned as usual, but get the
  FlagLocalReceiveOnly bit set.

- When changes are sent to the cluster this bit gets converted to the
  Invalid bit (like all other local flags, currently) and also the Version
  gets set to the empty version. The reason for clearing the Version is to
  ensure that other devices will not consider themselves out of date due to
  our change.

- The database layer accounts sizes per flag bit, so we can know how many
  files have been changed locally. We use this to trigger a "Revert" option
  on the folder when the amount of locally changed data is nonzero.

- To revert we take the files which have changed and reset their version
  counter down to zero. The next pull will replace our changed version with
  the globally latest. As this is a user-initiated operation we do not cause
  conflict copies when reverting.

- When pulling normally (i.e., not in the revert case) with local changes,
  normal conflict resolution will apply. Conflict copies will be created,
  but not propagated outwards (because receive only, right).

Implementation wise a receiveOnlyFolder is just a sendReceiveFolder that
sets an extra bit on local changes and has a Revert method.
*/
type receiveOnlyFolder struct {
	*sendReceiveFolder
}

func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, evLogger events.Logger, ioLimiter *util.Semaphore) service {
	sr := newSendReceiveFolder(model, fset, ignores, cfg, ver, evLogger, ioLimiter).(*sendReceiveFolder)
	sr.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files
	return &receiveOnlyFolder{sr}
}

func (f *receiveOnlyFolder) Revert() {
	f.doInSync(f.revert)
}

func (f *receiveOnlyFolder) revert() error {
	l.Infof("Reverting folder %v", f.Description())

	f.setState(FolderScanning)
	defer f.setState(FolderIdle)

	scanChan := make(chan string)
	go f.pullScannerRoutine(scanChan)
	defer close(scanChan)

	delQueue := &deleteQueue{
		handler:  f, // for the deleteItemOnDisk and deleteDirOnDisk methods
		ignores:  f.ignores,
		scanChan: scanChan,
	}

	batch := db.NewFileInfoBatch(func(files []protocol.FileInfo) error {
		f.updateLocalsFromScanning(files)
		return nil
	})
	snap, err := f.dbSnapshot()
	if err != nil {
		return err
	}
	defer snap.Release()
	snap.WithHave(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
		fi := intf.(protocol.FileInfo)
		if !fi.IsReceiveOnlyChanged() {
			// We're only interested in files that have changed locally in
			// receive only mode.
			return true
		}

		fi.LocalFlags &^= protocol.FlagLocalReceiveOnly

		switch gf, ok := snap.GetGlobal(fi.Name); {
		case !ok:
			msg := "Unexpected global file that we have locally"
			l.Debugf("%v revert: %v: %v", f, msg, fi.Name)
			f.evLogger.Log(events.Failure, msg)
			return true
		case gf.IsReceiveOnlyChanged():
			// The global file is our own. A revert then means to delete it.
			// We'll delete files directly, directories get queued and
			// handled below.
			if fi.Deleted {
				fi.Version = protocol.Vector{} // if this file ever resurfaces anywhere we want our delete to be strictly older
				break
			}
			handled, err := delQueue.handle(fi, snap)
			if err != nil {
				l.Infof("Revert: deleting %s: %v\n", fi.Name, err)
				return true // continue
			}
			if !handled {
				return true // continue
			}
			fi.SetDeleted(f.shortID)
			fi.Version = protocol.Vector{} // if this file ever resurfaces anywhere we want our delete to be strictly older
		case gf.IsEquivalentOptional(fi, f.modTimeWindow, false, false, protocol.FlagLocalReceiveOnly):
			// What we have locally is equivalent to the global file.
			fi = gf
		default:
			// Revert means to throw away our local changes. We reset the
			// version to the empty vector, which is strictly older than any
			// other existing version. It is not in conflict with anything,
			// either, so we will not create a conflict copy of our local
			// changes.
			fi.Version = protocol.Vector{}
		}

		batch.Append(fi)
		_ = batch.FlushIfFull()

		return true
	})
	_ = batch.Flush()

	// Handle any queued directories
	deleted, err := delQueue.flush(snap)
	if err != nil {
		l.Infoln("Revert:", err)
	}
	now := time.Now()
	for _, dir := range deleted {
		batch.Append(protocol.FileInfo{
			Name:       dir,
			Type:       protocol.FileInfoTypeDirectory,
			ModifiedS:  now.Unix(),
			ModifiedBy: f.shortID,
			Deleted:    true,
			Version:    protocol.Vector{},
		})
	}
	_ = batch.Flush()

	// We will likely have changed our local index, but that won't trigger a
	// pull by itself. Make sure we schedule one so that we start
	// downloading files.
	f.SchedulePull()

	return nil
}

// deleteQueue handles deletes by delegating to a handler and queuing
// directories for last.
type deleteQueue struct {
	handler interface {
		deleteItemOnDisk(item protocol.FileInfo, snap *db.Snapshot, scanChan chan<- string) error
		deleteDirOnDisk(dir string, snap *db.Snapshot, scanChan chan<- string) error
	}
	ignores  *ignore.Matcher
	dirs     []string
	scanChan chan<- string
}

func (q *deleteQueue) handle(fi protocol.FileInfo, snap *db.Snapshot) (bool, error) {
	// Things that are ignored but not marked deletable are not processed.
	ign := q.ignores.Match(fi.Name)
	if ign.IsIgnored() && !ign.IsDeletable() {
		return false, nil
	}

	// Directories are queued for later processing.
	if fi.IsDirectory() {
		q.dirs = append(q.dirs, fi.Name)
		return false, nil
	}

	// Kill it.
	err := q.handler.deleteItemOnDisk(fi, snap, q.scanChan)
	return true, err
}

func (q *deleteQueue) flush(snap *db.Snapshot) ([]string, error) {
	// Process directories from the leaves inward.
	sort.Sort(sort.Reverse(sort.StringSlice(q.dirs)))

	var firstError error
	var deleted []string

	for _, dir := range q.dirs {
		if err := q.handler.deleteDirOnDisk(dir, snap, q.scanChan); err == nil {
			deleted = append(deleted, dir)
		} else if err != nil && firstError == nil {
			firstError = err
		}
	}

	return deleted, firstError
}
<|endoftext|>


<commit_before>// Copyright 2012 Gonéri Le Bouder. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "notmuch"
import "log"
import "encoding/json"
import "os"
import "io"
import "fmt"
import "regexp"
import "net/mail"
import "path"

type Filter struct {
	Field   string
	Pattern string
	Re      *regexp.Regexp
	Tags    string
}

type Result struct {
	MessageID string
	Tags      string
	Die       bool
        Filename  string
}

const NCPU = 4 // number of CPU cores

func getMaildirLoc() string {
	// honor NOTMUCH_CONFIG
	home := os.Getenv("NOTMUCH_CONFIG")
	if home == "" {
		home = os.Getenv("HOME")
	}

	return path.Join(home, "Maildir")
}

func RefreshFlags(nmdb *notmuch.Database) {

	query := nmdb.CreateQuery("tag:inbox and tag:delete")
	msgs := query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:archive")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and not tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.AddTag("archive")
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:bug")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:killed")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.RemoveTag("inbox")
		}
	}

	query = nmdb.CreateQuery("tag:inbox")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.AddTag("inbox")
		}
	}

	nmdb.Close()
	fmt.Print("Ok\n")

}

func studyMsg(filter []Filter, filenameIn chan string, resultOut chan Result, quit chan bool) {
	for {
		filename := <-filenameIn

		if filename == "" {
			var result Result
			result.Die = true
			resultOut <- result

			return
		}
		// We can use Notmuch for this directly because Xappian will
		// fails as soon as we have 2 concurrent goroutine
		file, err := os.Open(filename) // For read access.
		if err != nil {
			log.Fatal(err)
		}
		var msg *mail.Message
		msg, err = mail.ReadMessage(file)

		if err != nil {
			log.Fatal(err)
		}

		var result Result
		result.MessageID = msg.Header.Get("Message-Id")
		result.Filename = filename
		for _, f := range filter {
			if f.Re.MatchString(msg.Header.Get(f.Field)) {
				result.Tags += " "
				result.Tags += f.Tags
			}

		}
		file.Close()

		resultOut <- result
	}
}

func loadFilter() (filter []Filter) {

	file, err := os.Open(fmt.Sprintf("/%s/notmuch-filter.json", getMaildirLoc())) // For read access.
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	dec := json.NewDecoder(file)
	for {
		var f Filter
		if err := dec.Decode(&f); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		var err error = nil
		if f.Re, err = regexp.Compile(f.Pattern); err != nil {
			log.Printf("error: %v\n", err)
		}

		filter = append(filter, f)
	}

	return filter
}

func studyMsgs(resultOut chan Result, quit chan bool, filenames []string) {

	filter := loadFilter()

	filenameIn := make(chan string)
	for i := 0; i < NCPU+1; i++ {
		go studyMsg(filter, filenameIn, resultOut, quit)
	}
	for _, filename := range filenames {
		filenameIn <- filename
	}

	for i := 0; i < NCPU+1; i++ {
		filenameIn <- ""
	}

	quit <- true
}

func main() {
	var query *notmuch.Query
	var nmdb *notmuch.Database

	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		notmuch.DATABASE_MODE_READ_ONLY); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}

	quit := make(chan bool)
	resultOut := make(chan Result)

	query = nmdb.CreateQuery("tag:new")

	println(">", query.CountMessages(), "<")
	msgs := query.SearchMessages()

	var filenames []string
	if query.CountMessages() > 0 {
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()

			filenames = append(filenames, msg.GetFileName())
		}
	}

	go studyMsgs(resultOut, quit, filenames)

	//	var query *notmuch.Query
	var msgIDRegexp = regexp.MustCompile("^<(.*)>$")
	var tagRegexp = regexp.MustCompile("([\\+-])(\\S+)")

	// open the database
	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		1); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}
	defer nmdb.Close()

	var running int = NCPU + 1
	for {
		result := <-resultOut

		if result.Die {

			running--

			if running > 0 {
				continue
			} else {
				RefreshFlags(nmdb)
				os.Exit(0)
			}
		}

		// Message-ID without the <>
		msgID := msgIDRegexp.FindStringSubmatch(result.MessageID)[1]
		filter := "id:"
		filter += msgID
		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		msg := msgs.Get()
		if msg == nil {
                    fmt.Printf("Can't find MessageID %s for mail %s\n", msgID, result.Filename)
                    return;
                }

		fmt.Printf("%s, tags: %s\n", msgID, result.Tags)
		msg.Freeze()
		for _, v := range tagRegexp.FindAllStringSubmatch(result.Tags, -1) {
			if v[1] == "+" {
				msg.AddTag(v[2])
			} else if v[1] == "-" {
				msg.RemoveTag(v[2])
			}
		}
		msg.Thaw()

	}

}
<commit_msg>skip mail with no MessageId<commit_after>// Copyright 2012 Gonéri Le Bouder. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "notmuch"
import "log"
import "encoding/json"
import "os"
import "io"
import "fmt"
import "regexp"
import "net/mail"
import "path"

type Filter struct {
	Field   string
	Pattern string
	Re      *regexp.Regexp
	Tags    string
}

type Result struct {
	MessageID string
	Tags      string
	Die       bool
        Filename  string
}

const NCPU = 4 // number of CPU cores

func getMaildirLoc() string {
	// honor NOTMUCH_CONFIG
	home := os.Getenv("NOTMUCH_CONFIG")
	if home == "" {
		home = os.Getenv("HOME")
	}

	return path.Join(home, "Maildir")
}

func RefreshFlags(nmdb *notmuch.Database) {

	query := nmdb.CreateQuery("tag:inbox and tag:delete")
	msgs := query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:archive")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and not tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.AddTag("archive")
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:list")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:inbox and tag:seen and tag:bug")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		msg.RemoveTag("inbox")
	}

	query = nmdb.CreateQuery("tag:killed")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.RemoveTag("inbox")
		}
	}

	query = nmdb.CreateQuery("tag:inbox")
	msgs = query.SearchMessages()
	for ; msgs.Valid(); msgs.MoveToNext() {
		msg := msgs.Get()
		threadId := msg.GetThreadId()
		filter := fmt.Sprintf("thread:%s", threadId)

		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()
			msg.AddTag("inbox")
		}
	}

	nmdb.Close()
	fmt.Print("Ok\n")

}

func studyMsg(filter []Filter, filenameIn chan string, resultOut chan Result, quit chan bool) {
	for {
		filename := <-filenameIn

		if filename == "" {
			var result Result
			result.Die = true
			resultOut <- result

			return
		}
		// We can use Notmuch for this directly because Xappian will
		// fails as soon as we have 2 concurrent goroutine
		file, err := os.Open(filename) // For read access.
		if err != nil {
			log.Fatal(err)
		}
		var msg *mail.Message
		msg, err = mail.ReadMessage(file)

		if err != nil {
			log.Fatal(err)
		}

		var result Result
		result.MessageID = msg.Header.Get("Message-Id")
                if (result.MessageID == "") {
                    fmt.Printf("No message ID for %s\n", filename)
                    continue;
                }
		result.Filename = filename
		for _, f := range filter {
			if f.Re.MatchString(msg.Header.Get(f.Field)) {
				result.Tags += " "
				result.Tags += f.Tags
			}

		}
		file.Close()

		resultOut <- result
	}
}

func loadFilter() (filter []Filter) {

	file, err := os.Open(fmt.Sprintf("/%s/notmuch-filter.json", getMaildirLoc())) // For read access.
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	dec := json.NewDecoder(file)
	for {
		var f Filter
		if err := dec.Decode(&f); err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		var err error = nil
		if f.Re, err = regexp.Compile(f.Pattern); err != nil {
			log.Printf("error: %v\n", err)
		}

		filter = append(filter, f)
	}

	return filter
}

func studyMsgs(resultOut chan Result, quit chan bool, filenames []string) {

	filter := loadFilter()

	filenameIn := make(chan string)
	for i := 0; i < NCPU+1; i++ {
		go studyMsg(filter, filenameIn, resultOut, quit)
	}
	for _, filename := range filenames {
		filenameIn <- filename
	}

	for i := 0; i < NCPU+1; i++ {
		filenameIn <- ""
	}

	quit <- true
}

func main() {
	var query *notmuch.Query
	var nmdb *notmuch.Database

	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		notmuch.DATABASE_MODE_READ_ONLY); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}

	quit := make(chan bool)
	resultOut := make(chan Result)

	query = nmdb.CreateQuery("tag:new")

	println(">", query.CountMessages(), "<")
	msgs := query.SearchMessages()

	var filenames []string
	if query.CountMessages() > 0 {
		for ; msgs.Valid(); msgs.MoveToNext() {
			msg := msgs.Get()

			filenames = append(filenames, msg.GetFileName())
		}
	}

	go studyMsgs(resultOut, quit, filenames)

	//	var query *notmuch.Query
	var msgIDRegexp = regexp.MustCompile("^<(.*)>$")
	var tagRegexp = regexp.MustCompile("([\\+-])(\\S+)")

	// open the database
	if db, status := notmuch.OpenDatabase(getMaildirLoc(),
		1); status == notmuch.STATUS_SUCCESS {
		nmdb = db
	} else {
		log.Fatalf("Failed to open the database: %v\n", status)
	}
	defer nmdb.Close()

	var running int = NCPU + 1
	for {
		result := <-resultOut

		if result.Die {

			running--

			if running > 0 {
				continue
			} else {
				RefreshFlags(nmdb)
				os.Exit(0)
			}
		}

		// Message-ID without the <>
		msgID := msgIDRegexp.FindStringSubmatch(result.MessageID)[1]
		filter := "id:"
		filter += msgID
		query := nmdb.CreateQuery(filter)
		msgs := query.SearchMessages()
		msg := msgs.Get()
		if msg == nil {
                    fmt.Printf("Can't find MessageID %s for mail %s\n", msgID, result.Filename)
                    return;
                }

		fmt.Printf("%s, tags: %s\n", msgID, result.Tags)
		msg.Freeze()
		for _, v := range tagRegexp.FindAllStringSubmatch(result.Tags, -1) {
			if v[1] == "+" {
				msg.AddTag(v[2])
			} else if v[1] == "-" {
				msg.RemoveTag(v[2])
			}
		}
		msg.Thaw()

	}

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"io"
	"os/exec"
	"strings"
	"time"
)

type Ntpdate struct {
	measurements chan<- *Measurement
	Servers      []string
}

func NewNtpdatePoller(measurements chan<- *Measurement, config Config) Ntpdate {
	return Ntpdate{
		measurements: measurements,
		Servers:      config.NtpdateServers,
	}
}

//FIXME: Timeout
func (poller Ntpdate) Poll(tick time.Time) {
	ctx := Slog{"poller": poller.Name(), "fn": "Poll", "tick": tick}

	if len(poller.Servers) > 0 {
		cmd := exec.Command("ntpdate", "-q", "-u")
		cmd.Args = append(cmd.Args, poller.Servers...)
		stdout, err := cmd.StdoutPipe()
		if err != nil {
			ctx.Error(err, "creating stdout pipe")
			poller.measurements <- &Measurement{tick, poller.Name(), []string{"error"}, float64(1)}
			return
		}

		if err := cmd.Start(); err != nil {
			ctx.Error(err, "starting sub command")
			poller.measurements <- &Measurement{tick, poller.Name(), []string{"error"}, float64(1)}
			return
		}

		defer func() {
			if err := cmd.Wait(); err != nil {
				poller.measurements <- &Measurement{tick, poller.Name(), []string{"error"}, float64(1)}
				ctx.Error(err, "waiting for subcommand to end")
			}
		}()

		buf := bufio.NewReader(stdout)

		for {
			line, err := buf.ReadString('\n')
			if err == nil {
				if strings.HasPrefix(line, "server") {
					parts := strings.Split(line, ",")
					server := strings.Replace(strings.Fields(parts[0])[1], ".", "_", 4)
					offset := strings.Fields(parts[2])[1]
					delay := strings.Fields(parts[3])[1]
					poller.measurements <- &Measurement{tick, poller.Name(), []string{"offset", server}, Atofloat64(offset)}
					poller.measurements <- &Measurement{tick, poller.Name(), []string{"delay", server}, Atofloat64(delay)}
				}
			} else {
				if err == io.EOF {
					break
				} else {
					ctx.Error(err, "unknown error reading data from subcommand")
					poller.measurements <- &Measurement{tick, poller.Name(), []string{"error"}, float64(1)}
					return
				}
			}
		}

	}
}

func (poller Ntpdate) Name() string {
	return "ntpdate"
}

func (poller Ntpdate) Exit() {}
<commit_msg>Consistent ordering of calls<commit_after>package main

import (
	"bufio"
	"io"
	"os/exec"
	"strings"
	"time"
)

type Ntpdate struct {
	measurements chan<- *Measurement
	Servers      []string
}

func NewNtpdatePoller(measurements chan<- *Measurement, config Config) Ntpdate {
	return Ntpdate{
		measurements: measurements,
		Servers:      config.NtpdateServers,
	}
}

//FIXME: Timeout
func (poller Ntpdate) Poll(tick time.Time) {
	ctx := Slog{"poller": poller.Name(), "fn": "Poll", "tick": tick}

	if len(poller.Servers) > 0 {
		cmd := exec.Command("ntpdate", "-q", "-u")
		cmd.Args = append(cmd.Args, poller.Servers...)
		stdout, err := cmd.StdoutPipe()
		if err != nil {
			ctx.Error(err, "creating stdout pipe")
			poller.measurements <- &Measurement{tick, poller.Name(), []string{"error"}, float64(1)}
			return
		}

		if err := cmd.Start(); err != nil {
			ctx.Error(err, "starting sub command")
			poller.measurements <- &Measurement{tick, poller.Name(), []string{"error"}, float64(1)}
			return
		}

		defer func() {
			if err := cmd.Wait(); err != nil {
				ctx.Error(err, "waiting for subcommand to end")
				poller.measurements <- &Measurement{tick, poller.Name(), []string{"error"}, float64(1)}
			}
		}()

		buf := bufio.NewReader(stdout)

		for {
			line, err := buf.ReadString('\n')
			if err == nil {
				if strings.HasPrefix(line, "server") {
					parts := strings.Split(line, ",")
					server := strings.Replace(strings.Fields(parts[0])[1], ".", "_", 4)
					offset := strings.Fields(parts[2])[1]
					delay := strings.Fields(parts[3])[1]
					poller.measurements <- &Measurement{tick, poller.Name(), []string{"offset", server}, Atofloat64(offset)}
					poller.measurements <- &Measurement{tick, poller.Name(), []string{"delay", server}, Atofloat64(delay)}
				}
			} else {
				if err == io.EOF {
					break
				} else {
					ctx.Error(err, "unknown error reading data from subcommand")
					poller.measurements <- &Measurement{tick, poller.Name(), []string{"error"}, float64(1)}
					return
				}
			}
		}

	}
}

func (poller Ntpdate) Name() string {
	return "ntpdate"
}

func (poller Ntpdate) Exit() {}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"sync"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/log"
)

const (
	namespace = "nginx" // For Prometheus metrics.
)

var (
	listeningAddress = flag.String("telemetry.address", ":9113", "Address on which to expose metrics.")
	metricsEndpoint  = flag.String("telemetry.endpoint", "/metrics", "Path under which to expose metrics.")
	nginxScrapeURI   = flag.String("nginx.scrape_uri", "http://localhost/nginx_status", "URI to nginx stub status page")
	insecure         = flag.Bool("insecure", true, "Ignore server certificate if using https")
)

// Exporter collects nginx stats from the given URI and exports them using
// the prometheus metrics package.
type Exporter struct {
	URI    string
	mutex  sync.RWMutex
	client *http.Client

	scrapeFailures       prometheus.Counter
	processedConnections *prometheus.CounterVec
	currentConnections   *prometheus.GaugeVec
}

// NewExporter returns an initialized Exporter.
func NewExporter(uri string) *Exporter {
	return &Exporter{
		URI: uri,
		scrapeFailures: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "exporter_scrape_failures_total",
			Help:      "Number of errors while scraping nginx.",
		}),
		processedConnections: prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "connections_processed_total",
			Help:      "Number of connections processed by nginx",
		},
			[]string{"stage"},
		),
		currentConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "connections_current",
			Help:      "Number of connections currently processed by nginx",
		},
			[]string{"state"},
		),
		client: &http.Client{
			Transport: &http.Transport{
				TLSClientConfig: &tls.Config{InsecureSkipVerify: *insecure},
			},
		},
	}
}

// Describe describes all the metrics ever exported by the nginx exporter. It
// implements prometheus.Collector.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	e.processedConnections.Describe(ch)
	e.currentConnections.Describe(ch)
	e.scrapeFailures.Describe(ch)
}

func (e *Exporter) collect(ch chan<- prometheus.Metric) error {
	resp, err := e.client.Get(e.URI)
	if err != nil {
		return fmt.Errorf("Error scraping nginx: %v", err)
	}

	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
		if err != nil {
			data = []byte(err.Error())
		}
		return fmt.Errorf("Status %s (%d): %s", resp.Status, resp.StatusCode, data)
	}

	// Parsing results
	lines := strings.Split(string(data), "\n")
	if len(lines) != 5 {
		return fmt.Errorf("Unexpected number of lines in status: %v", lines)
	}

	// active connections
	parts := strings.Split(lines[0], ":")
	if len(parts) != 2 {
		return fmt.Errorf("Unexpected first line: %s", lines[0])
	}
	v, err := strconv.Atoi(strings.TrimSpace(parts[1]))
	if err != nil {
		return err
	}
	e.currentConnections.WithLabelValues("active").Set(float64(v))

	// processed connections
	parts = strings.Fields(lines[2])
	if len(parts) != 3 {
		return fmt.Errorf("Unexpected third line: %s", lines[2])
	}
	v, err = strconv.Atoi(strings.TrimSpace(parts[0]))
	if err != nil {
		return err
	}
	e.processedConnections.WithLabelValues("accepted").Set(float64(v))
	v, err = strconv.Atoi(strings.TrimSpace(parts[1]))
	if err != nil {
		return err
	}
	e.processedConnections.WithLabelValues("handled").Set(float64(v))
	v, err = strconv.Atoi(strings.TrimSpace(parts[2]))
	if err != nil {
		return err
	}
	e.processedConnections.WithLabelValues("any").Set(float64(v))

	// current connections
	parts = strings.Fields(lines[3])
	if len(parts) != 6 {
		return fmt.Errorf("Unexpected fourth line: %s", lines[3])
	}
	v, err = strconv.Atoi(strings.TrimSpace(parts[1]))
	if err != nil {
		return err
	}
	e.currentConnections.WithLabelValues("reading").Set(float64(v))
	v, err = strconv.Atoi(strings.TrimSpace(parts[3]))
	if err != nil {
		return err
	}

	e.currentConnections.WithLabelValues("writing").Set(float64(v))
	v, err = strconv.Atoi(strings.TrimSpace(parts[5]))
	if err != nil {
		return err
	}
	e.currentConnections.WithLabelValues("waiting").Set(float64(v))
	return nil
}

// Collect fetches the stats from configured nginx location and delivers them
// as Prometheus metrics. It implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	e.mutex.Lock() // To protect metrics from concurrent collects.
	defer e.mutex.Unlock()
	if err := e.collect(ch); err != nil {
		log.Printf("Error scraping nginx: %s", err)
		e.scrapeFailures.Inc()
		e.scrapeFailures.Collect(ch)
	}
	e.processedConnections.Collect(ch)
	e.currentConnections.Collect(ch)
	return
}

func main() {
	flag.Parse()

	exporter := NewExporter(*nginxScrapeURI)
	prometheus.MustRegister(exporter)

	log.Printf("Starting Server: %s", *listeningAddress)
	http.Handle(*metricsEndpoint, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
			<title>Nginx Exporter</title>

Nginx Exporter

<a href="` + *metricsEndpoint + `">Metrics

			`))
	})

	log.Fatal(http.ListenAndServe(*listeningAddress, nil))
}
<commit_msg>Change NewCounter with MustNewConstMetric()<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"sync"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/log"
)

const (
	namespace = "nginx" // For Prometheus metrics.
)

var (
	listeningAddress = flag.String("telemetry.address", ":9113", "Address on which to expose metrics.")
	metricsEndpoint  = flag.String("telemetry.endpoint", "/metrics", "Path under which to expose metrics.")
	nginxScrapeURI   = flag.String("nginx.scrape_uri", "http://localhost/nginx_status", "URI to nginx stub status page")
	insecure         = flag.Bool("insecure", true, "Ignore server certificate if using https")
)

// Exporter collects nginx stats from the given URI and exports them using
// the prometheus metrics package.
type Exporter struct {
	URI    string
	mutex  sync.RWMutex
	client *http.Client

	scrapeFailures       prometheus.Counter
	processedConnections *prometheus.Desc
	currentConnections   *prometheus.GaugeVec
}

// NewExporter returns an initialized Exporter.
func NewExporter(uri string) *Exporter {
	return &Exporter{
		URI: uri,
		scrapeFailures: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "exporter_scrape_failures_total",
			Help:      "Number of errors while scraping nginx.",
		}),
		processedConnections: prometheus.NewDesc(
                        prometheus.BuildFQName(namespace, "", "connections_processed_total"),
			"Number of connections processed by nginx",
                        []string{"accepted", "handled", "any"},
                        nil,
		),
		currentConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "connections_current",
			Help:      "Number of connections currently processed by nginx",
		},
			[]string{"state"},
		),
		client: &http.Client{
			Transport: &http.Transport{
				TLSClientConfig: &tls.Config{InsecureSkipVerify: *insecure},
			},
		},
	}
}

// Describe describes all the metrics ever exported by the nginx exporter. It
// implements prometheus.Collector.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- e.processedConnections
	e.currentConnections.Describe(ch)
	e.scrapeFailures.Describe(ch)
}

func (e *Exporter) collect(ch chan<- prometheus.Metric) error {
	resp, err := e.client.Get(e.URI)
	if err != nil {
		return fmt.Errorf("Error scraping nginx: %v", err)
	}

	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
		if err != nil {
			data = []byte(err.Error())
		}
		return fmt.Errorf("Status %s (%d): %s", resp.Status, resp.StatusCode, data)
	}

	// Parsing results
	lines := strings.Split(string(data), "\n")
	if len(lines) != 5 {
		return fmt.Errorf("Unexpected number of lines in status: %v", lines)
	}

	// active connections
	parts := strings.Split(lines[0], ":")
	if len(parts) != 2 {
		return fmt.Errorf("Unexpected first line: %s", lines[0])
	}
	v, err := strconv.Atoi(strings.TrimSpace(parts[1]))
	if err != nil {
		return err
	}
	e.currentConnections.WithLabelValues("active").Set(float64(v))

	// processed connections
	parts = strings.Fields(lines[2])
	if len(parts) != 3 {
		return fmt.Errorf("Unexpected third line: %s", lines[2])
	}
	v, err = strconv.Atoi(strings.TrimSpace(parts[0]))
	if err != nil {
		return err
	}
        ch <- prometheus.MustNewConstMetric(e.processedConnections, prometheus.CounterValue, float64(v), "accepted")
	v, err = strconv.Atoi(strings.TrimSpace(parts[1]))
	if err != nil {
		return err
	}
        ch <- prometheus.MustNewConstMetric(e.processedConnections, prometheus.CounterValue, float64(v), "handled")
	v, err = strconv.Atoi(strings.TrimSpace(parts[2]))
	if err != nil {
		return err
	}
        ch <- prometheus.MustNewConstMetric(e.processedConnections, prometheus.CounterValue, float64(v), "any")

	// current connections
	parts = strings.Fields(lines[3])
	if len(parts) != 6 {
		return fmt.Errorf("Unexpected fourth line: %s", lines[3])
	}
	v, err = strconv.Atoi(strings.TrimSpace(parts[1]))
	if err != nil {
		return err
	}
	e.currentConnections.WithLabelValues("reading").Set(float64(v))
	v, err = strconv.Atoi(strings.TrimSpace(parts[3]))
	if err != nil {
		return err
	}

	e.currentConnections.WithLabelValues("writing").Set(float64(v))
	v, err = strconv.Atoi(strings.TrimSpace(parts[5]))
	if err != nil {
		return err
	}
	e.currentConnections.WithLabelValues("waiting").Set(float64(v))
	return nil
}

// Collect fetches the stats from configured nginx location and delivers them
// as Prometheus metrics. It implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	e.mutex.Lock() // To protect metrics from concurrent collects.
	defer e.mutex.Unlock()
	if err := e.collect(ch); err != nil {
		log.Printf("Error scraping nginx: %s", err)
		e.scrapeFailures.Inc()
		e.scrapeFailures.Collect(ch)
	}
	return
}

func main() {
	flag.Parse()

	exporter := NewExporter(*nginxScrapeURI)
	prometheus.MustRegister(exporter)

	log.Printf("Starting Server: %s", *listeningAddress)
	http.Handle(*metricsEndpoint, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
			<title>Nginx Exporter</title>

Nginx Exporter

<a href="` + *metricsEndpoint + `">Metrics

			`))
	})

	log.Fatal(http.ListenAndServe(*listeningAddress, nil))
}
<|endoftext|>