repo_description: Prometheus Operator creates/configures/manages Prometheus clusters atop Kubernetes
repo_topics: hacktoberfest kubernetes monitoring prometheus
code: // Copyright The prometheus-operator Authors
  //
  // Licensed under the Apache License, Version 2.0 (the "License");
  // you may not use this file except in compliance with the License.
  // You may obtain a copy of the License at
  //
  //     http://www.apache.org/licenses/LICENSE-2.0
  //
  // Unless required by applicable law or agreed to in writing, software
  // distributed under the License is distributed on an "AS IS" BASIS,
  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  // See the License for the specific language governing permissions and
  // limitations under the License.
  
  package alertmanager
  
  import (
  	"bytes"
  	"context"
  	"errors"
  	"fmt"
  	"log/slog"
  	"maps"
  	"path"
  	"strings"
  	"time"
  
  	"github.com/blang/semver/v4"
  	"github.com/mitchellh/hashstructure"
  	"github.com/prometheus/client_golang/prometheus"
  	"github.com/prometheus/common/model"
  	appsv1 "k8s.io/api/apps/v1"
  	corev1 "k8s.io/api/core/v1"
  	apierrors "k8s.io/apimachinery/pkg/api/errors"
  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  	"k8s.io/apimachinery/pkg/labels"
  	"k8s.io/client-go/kubernetes"
  	typedauthv1 "k8s.io/client-go/kubernetes/typed/authorization/v1"
  	"k8s.io/client-go/metadata"
  	"k8s.io/client-go/rest"
  	"k8s.io/client-go/tools/cache"
  	"k8s.io/utils/ptr"
  
  	"github.com/prometheus-operator/prometheus-operator/pkg/alertmanager/clustertlsconfig"
  	"github.com/prometheus-operator/prometheus-operator/pkg/alertmanager/validation"
  	validationv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/alertmanager/validation/v1alpha1"
  	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
  	monitoringv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1"
  	"github.com/prometheus-operator/prometheus-operator/pkg/assets"
  	monitoringv1ac "github.com/prometheus-operator/prometheus-operator/pkg/client/applyconfiguration/monitoring/v1"
  	monitoringclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
  	"github.com/prometheus-operator/prometheus-operator/pkg/informers"
  	"github.com/prometheus-operator/prometheus-operator/pkg/k8s"
  	"github.com/prometheus-operator/prometheus-operator/pkg/listwatch"
  	"github.com/prometheus-operator/prometheus-operator/pkg/operator"
  	"github.com/prometheus-operator/prometheus-operator/pkg/webconfig"
  )
  
  const (
  	resyncPeriod              = 5 * time.Minute
  	controllerName            = "alertmanager-controller"
  	applicationNameLabelValue = "alertmanager"
  
  	selectingAlertmanagerConfigResourcesAction = "SelectingAlertmanagerConfigResources"
  )
  
  // Config defines the operator's parameters for the Alertmanager controller.
  // Whenever the value of one of these parameters is changed, it triggers an
  // update of the managed statefulsets.
  type Config struct {
  	LocalHost                    string
  	ClusterDomain                string
  	ReloaderConfig               operator.ContainerConfig
  	AlertmanagerDefaultBaseImage string
  	Annotations                  operator.Map
  	Labels                       operator.Map
  }
  
  // Operator manages the lifecycle of the Alertmanager statefulsets and their
  // configurations.
  type Operator struct {
  	kclient    kubernetes.Interface
  	mdClient   metadata.Interface
  	mclient    monitoringclient.Interface
  	ssarClient typedauthv1.SelfSubjectAccessReviewInterface
  
  	controllerID string
  	repairPolicy operator.RepairPolicy
  
  	logger   *slog.Logger
  	accessor *operator.Accessor
  
  	nsAlrtInf    cache.SharedIndexInformer
  	nsAlrtCfgInf cache.SharedIndexInformer
  
  	alrtInfs    *informers.ForResource
  	alrtCfgInfs *informers.ForResource
  	cmapInfs    *informers.ForResource
  	secrInfs    *informers.ForResource
  	ssetInfs    *informers.ForResource
  
  	rr *operator.ResourceReconciler
  
  	metrics         *operator.Metrics
  	reconciliations *operator.ReconciliationTracker
  
  	newEventRecorder operator.NewEventRecorderFunc
  
  	canReadStorageClass bool
  
  	config Config
  
  	configResourcesStatusEnabled bool
  }
  
  type ControllerOption func(*Operator)
  
  // WithStorageClassValidation tells that the controller should verify that the
  // Prometheus spec references a valid StorageClass name.
  func WithStorageClassValidation() ControllerOption {
  	return func(o *Operator) {
  		o.canReadStorageClass = true
  	}
  }
  
  // WithConfigResourceStatus tells that the controller can manage the status of
  // configuration resources.
  func WithConfigResourceStatus() ControllerOption {
  	return func(o *Operator) {
  		o.configResourcesStatusEnabled = true
  	}
  }
  
  // New creates a new controller.
  func New(ctx context.Context, restConfig *rest.Config, c operator.Config, logger *slog.Logger, r prometheus.Registerer, options ...ControllerOption) (*Operator, error) {
  	logger = logger.With("component", controllerName)
  
  	client, err := kubernetes.NewForConfig(restConfig)
  	if err != nil {
  		return nil, fmt.Errorf("instantiating kubernetes client failed: %w", err)
  	}
  
  	mdClient, err := metadata.NewForConfig(restConfig)
  	if err != nil {
  		return nil, fmt.Errorf("instantiating kubernetes client failed: %w", err)
  	}
  
  	mclient, err := monitoringclient.NewForConfig(restConfig)
  	if err != nil {
  		return nil, fmt.Errorf("instantiating monitoring client failed: %w", err)
  	}
  
  	// All the metrics exposed by the controller get the controller="alertmanager" label.
  	r = prometheus.WrapRegistererWith(prometheus.Labels{"controller": "alertmanager"}, r)
  
  	o := &Operator{
  		kclient:    client,
  		mdClient:   mdClient,
  		mclient:    mclient,
  		ssarClient: client.AuthorizationV1().SelfSubjectAccessReviews(),
  
  		logger:   logger,
  		accessor: operator.NewAccessor(logger),
  
  		metrics:          operator.NewMetrics(r),
  		reconciliations:  &operator.ReconciliationTracker{},
  		newEventRecorder: c.EventRecorderFactory(client, controllerName),
  
  		controllerID: c.ControllerID,
  		repairPolicy: c.RepairPolicy,
  
  		config: Config{
  			LocalHost:                    c.LocalHost,
  			ClusterDomain:                c.ClusterDomain,
  			ReloaderConfig:               c.ReloaderConfig,
  			AlertmanagerDefaultBaseImage: c.AlertmanagerDefaultBaseImage,
  			Annotations:                  c.Annotations,
  			Labels:                       c.Labels,
  		},
  	}
  	for _, opt := range options {
  		opt(o)
  	}
  
  	if err := o.bootstrap(ctx, c); err != nil {
  		return nil, err
  	}
  
  	o.rr = operator.NewResourceReconciler(
  		o.logger,
  		o,
  		o.alrtInfs,
  		o.metrics,
  		monitoringv1.AlertmanagersKind,
  		r,
  		o.controllerID,
  	)
  
  	return o, nil
  }
  
  func (c *Operator) bootstrap(ctx context.Context, config operator.Config) error {
  	c.metrics.MustRegister(c.reconciliations)
  
  	var err error
  	c.alrtInfs, err = informers.NewInformersForResource(
  		informers.NewMonitoringInformerFactories(
  			config.Namespaces.AlertmanagerAllowList,
  			config.Namespaces.DenyList,
  			c.mclient,
  			resyncPeriod,
  			func(options *metav1.ListOptions) {
  				options.LabelSelector = config.AlertmanagerSelector.String()
  			},
  		),
  		monitoringv1.SchemeGroupVersion.WithResource(monitoringv1.AlertmanagerName),
  	)
  	if err != nil {
  		return fmt.Errorf("error creating alertmanager informers: %w", err)
  	}
  
  	var alertmanagerStores []cache.Store
  	for _, informer := range c.alrtInfs.GetInformers() {
  		alertmanagerStores = append(alertmanagerStores, informer.Informer().GetStore())
  	}
  	c.metrics.MustRegister(newAlertmanagerCollectorForStores(alertmanagerStores...))
  
  	c.alrtCfgInfs, err = informers.NewInformersForResource(
  		informers.NewMonitoringInformerFactories(
  			config.Namespaces.AlertmanagerConfigAllowList,
  			config.Namespaces.DenyList,
  			c.mclient,
  			resyncPeriod,
  			nil,
  		),
  		monitoringv1alpha1.SchemeGroupVersion.WithResource(monitoringv1alpha1.AlertmanagerConfigName),
  	)
  	if err != nil {
  		return fmt.Errorf("error creating alertmanagerconfig informers: %w", err)
  	}
  
  	allowList := config.Namespaces.AlertmanagerConfigAllowList
  	if config.WatchObjectRefsInAllNamespaces {
  		allowList = operator.MergeAllowLists(
  			config.Namespaces.AlertmanagerAllowList,
  			config.Namespaces.AlertmanagerConfigAllowList,
  		)
  	}
  
  	c.secrInfs, err = informers.NewInformersForResourceWithTransform(
  		informers.NewMetadataInformerFactory(
  			allowList,
  			config.Namespaces.DenyList,
  			c.mdClient,
  			resyncPeriod,
  			func(options *metav1.ListOptions) {
  				options.FieldSelector = config.SecretListWatchFieldSelector.String()
  				options.LabelSelector = config.SecretListWatchLabelSelector.String()
  			},
  		),
  		corev1.SchemeGroupVersion.WithResource(string(corev1.ResourceSecrets)),
  		informers.PartialObjectMetadataStrip(operator.SecretGVK()),
  	)
  	if err != nil {
  		return fmt.Errorf("error creating secret informers: %w", err)
  	}
  
  	c.cmapInfs, err = informers.NewInformersForResourceWithTransform(
  		informers.NewMetadataInformerFactory(
  			allowList,
  			config.Namespaces.DenyList,
  			c.mdClient,
  			resyncPeriod,
  			func(options *metav1.ListOptions) {
  				options.FieldSelector = config.ConfigMapListWatchFieldSelector.String()
  				options.LabelSelector = config.ConfigMapListWatchLabelSelector.String()
  			},
  		),
  		corev1.SchemeGroupVersion.WithResource(string(corev1.ResourceConfigMaps)),
  		informers.PartialObjectMetadataStrip(operator.ConfigMapGVK()),
  	)
  	if err != nil {
  		return fmt.Errorf("error creating configmap informers: %w", err)
  	}
  
  	c.ssetInfs, err = informers.NewInformersForResource(
  		informers.NewKubeInformerFactories(
  			config.Namespaces.AlertmanagerAllowList,
  			config.Namespaces.DenyList,
  			c.kclient,
  			resyncPeriod,
  			func(options *metav1.ListOptions) {
  				options.LabelSelector = labelSelectorForStatefulSets()
  			},
  		),
  		appsv1.SchemeGroupVersion.WithResource("statefulsets"),
  	)
  	if err != nil {
  		return fmt.Errorf("error creating statefulset informers: %w", err)
  	}
  
  	newNamespaceInformer := func(o *Operator, allowList map[string]struct{}) (cache.SharedIndexInformer, error) {
  		lw, privileged, err := listwatch.NewNamespaceListWatchFromClient(
  			ctx,
  			o.logger,
  			config.KubernetesVersion,
  			o.kclient.CoreV1(),
  			o.ssarClient,
  			allowList,
  			config.Namespaces.DenyList,
  		)
  		if err != nil {
  			return nil, fmt.Errorf("failed to create namespace lister/watcher: %w", err)
  		}
  
  		c.logger.Debug("creating namespace informer", "privileged", privileged)
  		return cache.NewSharedIndexInformer(
  			o.metrics.NewInstrumentedListerWatcher(lw),
  			&corev1.Namespace{},
  			resyncPeriod,
  			cache.Indexers{},
  		), nil
  	}
  	c.nsAlrtCfgInf, err = newNamespaceInformer(c, config.Namespaces.AlertmanagerConfigAllowList)
  	if err != nil {
  		return err
  	}
  
  	if listwatch.IdenticalNamespaces(config.Namespaces.AlertmanagerConfigAllowList, config.Namespaces.AlertmanagerAllowList) {
  		c.nsAlrtInf = c.nsAlrtCfgInf
  	} else {
  		c.nsAlrtInf, err = newNamespaceInformer(c, config.Namespaces.AlertmanagerAllowList)
  		if err != nil {
  			return err
  		}
  	}
  
  	return nil
  }
  
  // waitForCacheSync waits for the informers' caches to be synced.
  func (c *Operator) waitForCacheSync(ctx context.Context) error {
  	for _, infs := range []struct {
  		name                 string
  		informersForResource *informers.ForResource
  	}{
  		{"Alertmanager", c.alrtInfs},
  		{"AlertmanagerConfig", c.alrtCfgInfs},
  		{"Secret", c.secrInfs},
  		{"ConfigMap", c.cmapInfs},
  		{"StatefulSet", c.ssetInfs},
  	} {
  		for _, inf := range infs.informersForResource.GetInformers() {
  			if !operator.WaitForNamedCacheSync(ctx, "alertmanager", c.logger.With("informer", infs.name), inf.Informer()) {
  				return fmt.Errorf("failed to sync cache for %s informer", infs.name)
  			}
  		}
  	}
  
  	for _, inf := range []struct {
  		name     string
  		informer cache.SharedIndexInformer
  	}{
  		{"AlertmanagerNamespace", c.nsAlrtInf},
  		{"AlertmanagerConfigNamespace", c.nsAlrtCfgInf},
  	} {
  		if !operator.WaitForNamedCacheSync(ctx, "alertmanager", c.logger.With("informer", inf.name), inf.informer) {
  			return fmt.Errorf("failed to sync cache for %s informer", inf.name)
  		}
  	}
  
  	c.logger.Info("successfully synced all caches")
  	return nil
  }
  
  // addHandlers adds the eventhandlers to the informers.
  func (c *Operator) addHandlers() {
  	c.alrtInfs.AddEventHandler(c.rr)
  
  	c.ssetInfs.AddEventHandler(c.rr)
  
  	c.alrtCfgInfs.AddEventHandler(operator.NewEventHandler(
  		c.logger,
  		c.accessor,
  		c.metrics,
  		monitoringv1alpha1.AlertmanagerConfigKind,
  		c.enqueueForNamespace,
  		operator.WithFilter(
  			operator.AnyFilter(
  				operator.GenerationChanged,
  				operator.LabelsChanged,
  			),
  		),
  	))
  
  	hasRefFunc := operator.HasReferenceFunc(
  		c.alrtInfs,
  		c.reconciliations,
  	)
  	c.secrInfs.AddEventHandler(operator.NewEventHandler(
  		c.logger,
  		c.accessor,
  		c.metrics,
  		operator.SecretGVK().Kind,
  		c.enqueueForNamespace,
  		operator.WithFilter(operator.ResourceVersionChanged),
  		operator.WithFilter(hasRefFunc),
  	))
  
  	c.cmapInfs.AddEventHandler(operator.NewEventHandler(
  		c.logger,
  		c.accessor,
  		c.metrics,
  		operator.ConfigMapGVK().Kind,
  		c.enqueueForNamespace,
  		operator.WithFilter(operator.ResourceVersionChanged),
  		operator.WithFilter(hasRefFunc),
  	))
  
  	// The controller needs to watch the namespaces in which the
  	// alertmanagerconfigs live because a label change on a namespace may
  	// trigger a configuration change.
  	// It doesn't need to watch on addition/deletion though because it's
  	// already covered by the event handlers on alertmanagerconfigs.
  	_, _ = c.nsAlrtCfgInf.AddEventHandler(cache.ResourceEventHandlerFuncs{
  		UpdateFunc: c.handleNamespaceUpdate,
  	})
  }
  
  // enqueueForNamespace enqueues all Alertmanager object keys that belong to the
  // given namespace or select objects in the given namespace.
  func (c *Operator) enqueueForNamespace(nsName string) {
  	nsObject, exists, err := c.nsAlrtCfgInf.GetStore().GetByKey(nsName)
  	if err != nil {
  		c.logger.Error(
  			"get namespace to enqueue Alertmanager instances failed",
  			"err", err,
  		)
  		return
  	}
  	if !exists {
  		c.logger.Error(fmt.Sprintf("get namespace to enqueue Alertmanager instances failed: namespace %q does not exist", nsName))
  		return
  	}
  	ns := nsObject.(*corev1.Namespace)
  
  	err = c.alrtInfs.ListAll(labels.Everything(), func(obj any) {
  		// Check for Alertmanager instances in the namespace.
  		am := obj.(*monitoringv1.Alertmanager)
  		if am.Namespace == nsName {
  			c.rr.EnqueueForReconciliation(am)
  			return
  		}
  
  		// Check for Alertmanager instances selecting AlertmanagerConfigs in
  		// the namespace.
  		acNSSelector, err := metav1.LabelSelectorAsSelector(am.Spec.AlertmanagerConfigNamespaceSelector)
  		if err != nil {
  			c.logger.Error(
  				fmt.Sprintf("failed to convert AlertmanagerConfigNamespaceSelector of %q to selector", am.Name),
  				"err", err,
  			)
  			return
  		}
  
  		if acNSSelector.Matches(labels.Set(ns.Labels)) {
  			c.rr.EnqueueForReconciliation(am)
  			return
  		}
  	})
  	if err != nil {
  		c.logger.Error(
  			"listing all Alertmanager instances from cache failed",
  			"err", err,
  		)
  	}
  }
  
  // Run the controller.
  func (c *Operator) Run(ctx context.Context) error {
  	go c.rr.Run(ctx)
  	defer c.rr.Stop()
  
  	go c.alrtInfs.Start(ctx.Done())
  	go c.alrtCfgInfs.Start(ctx.Done())
  	go c.secrInfs.Start(ctx.Done())
  	go c.cmapInfs.Start(ctx.Done())
  	go c.ssetInfs.Start(ctx.Done())
  	go c.nsAlrtCfgInf.Run(ctx.Done())
  	if c.nsAlrtInf != c.nsAlrtCfgInf {
  		go c.nsAlrtInf.Run(ctx.Done())
  	}
  
  	if err := c.waitForCacheSync(ctx); err != nil {
  		return err
  	}
  
  	// Refresh the status of the existing Alertmanager objects.
  	_ = c.alrtInfs.ListAll(labels.Everything(), func(obj any) {
  		c.RefreshStatusFor(obj.(*monitoringv1.Alertmanager))
  	})
  
  	c.addHandlers()
  
  	// TODO(simonpasquier): watch for Alertmanager pods instead of polling.
  	go operator.StatusPoller(ctx, c)
  
  	c.metrics.Ready().Set(1)
  	<-ctx.Done()
  	return nil
  }
  
  // Iterate implements the operator.StatusReconciler interface.
  func (c *Operator) Iterate(processFn func(operator.StatusGetter)) {
  	if err := c.alrtInfs.ListAll(labels.Everything(), func(o any) {
  		processFn(o.(*monitoringv1.Alertmanager))
  	}); err != nil {
  		c.logger.Error("failed to list Alertmanager objects", "err", err)
  	}
  }
  
  // RefreshStatusFor implements the operator.StatusReconciler interface.
  func (c *Operator) RefreshStatusFor(o metav1.Object) {
  	c.rr.EnqueueForStatus(o)
  }
  
  func alertmanagerKeyToStatefulSetKey(key string) string {
  	keyParts := strings.Split(key, "/")
  	return keyParts[0] + "/alertmanager-" + keyParts[1]
  }
  
  func (c *Operator) handleNamespaceUpdate(oldo, curo any) {
  	old := oldo.(*corev1.Namespace)
  	cur := curo.(*corev1.Namespace)
  
  	c.logger.Debug("update handler", "namespace", cur.GetName(), "old", old.ResourceVersion, "cur", cur.ResourceVersion)
  
  	// Periodic resync may resend the Namespace without changes
  	// in-between.
  	if old.ResourceVersion == cur.ResourceVersion {
  		return
  	}
  
  	c.logger.Debug("Namespace updated", "namespace", cur.GetName())
  	c.metrics.TriggerByCounter("Namespace", operator.UpdateEvent).Inc()
  
  	// Check for Alertmanager instances selecting AlertmanagerConfigs in the namespace.
  	err := c.alrtInfs.ListAll(labels.Everything(), func(obj any) {
  		a := obj.(*monitoringv1.Alertmanager)
  
  		sync, err := k8s.LabelSelectionHasChanged(old.Labels, cur.Labels, a.Spec.AlertmanagerConfigNamespaceSelector)
  		if err != nil {
  			c.logger.Error(
  				"failed to detect label selection change",
  				"err", err,
  				"name", a.Name,
  				"namespace", a.Namespace,
  			)
  			return
  		}
  
  		if sync {
  			c.rr.EnqueueForReconciliation(a)
  		}
  	})
  	if err != nil {
  		c.logger.Error(
  			"listing all Alertmanager instances from cache failed",
  			"err", err,
  		)
  	}
  }
  
  // Sync implements the operator.Syncer interface.
  func (c *Operator) Sync(ctx context.Context, key string) error {
  	c.reconciliations.ResetStatus(key)
  	err := c.sync(ctx, key)
  	c.reconciliations.SetStatus(key, err)
  
  	return err
  }
  
  func (c *Operator) sync(ctx context.Context, key string) error {
  	am, err := operator.GetObjectFromKey[*monitoringv1.Alertmanager](c.alrtInfs, key)
  	if err != nil {
  		return err
  	}
  
  	if am == nil {
  		c.reconciliations.ForgetObject(key)
  		// Dependent resources are cleaned up by K8s via OwnerReferences
  		return nil
  	}
  
  	// Check if the Alertmanager instance is marked for deletion.
  	if c.rr.DeletionInProgress(am) {
  		c.reconciliations.ForgetObject(key)
  		return nil
  	}
  
  	logger := c.logger.With("key", key)
  	logger.Info("sync alertmanager")
  
  	if am.Spec.Paused {
  		logger.Info("no action taken (the resource is paused)")
  		return nil
  	}
  
  	c.recordDeprecatedFields(key, logger, am)
  
  	if err := operator.CheckStorageClass(ctx, c.canReadStorageClass, c.kclient, am.Spec.Storage); err != nil {
  		return err
  	}
  
  	assetStore := assets.NewStoreBuilder(c.kclient.CoreV1(), c.kclient.CoreV1())
  
  	if err := c.provisionAlertmanagerConfiguration(ctx, am, assetStore); err != nil {
  		return fmt.Errorf("provision alertmanager configuration: %w", err)
  	}
  	c.reconciliations.UpdateReferenceTracker(key, assetStore.RefTracker())
  
  	tlsShardedSecret, err := operator.ReconcileShardedSecret(ctx, assetStore.TLSAssets(), c.kclient, c.newTLSAssetSecret(am))
  	if err != nil {
  		return fmt.Errorf("failed to reconcile the TLS secrets: %w", err)
  	}
  
  	if err := c.createOrUpdateWebConfigSecret(ctx, am); err != nil {
  		return fmt.Errorf("failed to synchronize the web config secret: %w", err)
  	}
  
  	// TODO(simonpasquier): the operator should take into account changes to
  	// the cluster TLS configuration to trigger a rollout of the pods (this
  	// configuration doesn't support live reload).
  	if err := c.createOrUpdateClusterTLSConfigSecret(ctx, am); err != nil {
  		return fmt.Errorf("failed to synchronize the cluster TLS config secret: %w", err)
  	}
  
  	svcClient := c.kclient.CoreV1().Services(am.Namespace)
  	if am.Spec.ServiceName != nil {
  		selectorLabels := makeSelectorLabels(am.Name)
  		if err := k8s.EnsureCustomGoverningService(ctx, am.Namespace, *am.Spec.ServiceName, svcClient, selectorLabels); err != nil {
  			return err
  		}
  	} else {
  		// Create governing service if it doesn't exist.
  		if _, err = k8s.CreateOrUpdateService(ctx, svcClient, makeStatefulSetService(am, c.config)); err != nil {
  			return fmt.Errorf("synchronizing governing service failed: %w", err)
  		}
  	}
  
  	existingStatefulSet, err := c.getStatefulSetFromAlertmanagerKey(key)
  	if err != nil {
  		return err
  	}
  
  	shouldCreate := false
  	if existingStatefulSet == nil {
  		shouldCreate = true
  		existingStatefulSet = &appsv1.StatefulSet{}
  	}
  
  	if c.rr.DeletionInProgress(existingStatefulSet) {
  		return nil
  	}
  
  	newSSetInputHash, err := createSSetInputHash(*am, c.config, tlsShardedSecret, existingStatefulSet.Spec)
  	if err != nil {
  		return err
  	}
  
  	sset, err := makeStatefulSet(logger, am, c.config, newSSetInputHash, tlsShardedSecret)
  	if err != nil {
  		return fmt.Errorf("failed to generate statefulset: %w", err)
  	}
  	operator.SanitizeSTS(sset)
  
  	if newSSetInputHash == existingStatefulSet.Annotations[operator.InputHashAnnotationKey] {
  		logger.Debug("new statefulset generation inputs match current, skipping any actions")
  		return nil
  	}
  
  	ssetClient := c.kclient.AppsV1().StatefulSets(am.Namespace)
  	if shouldCreate {
  		logger.Debug("no current statefulset found")
  		logger.Debug("creating statefulset")
  		if _, err := k8s.CreateStatefulSetOrPatchLabels(ctx, ssetClient, sset); err != nil {
  			return fmt.Errorf("failed to create statefulset: %w", err)
  		}
  		return nil
  	}
  
  	if err = k8s.ForceUpdateStatefulSet(ctx, ssetClient, sset, func(reason string) {
  		c.metrics.StsDeleteCreateCounter().Inc()
  		logger.Info("recreating StatefulSet because the update operation wasn't possible", "reason", reason)
  	}); err != nil {
  		return err
  	}
  
  	return nil
  }
  
  // getStatefulSetFromAlertmanagerKey returns a copy of the StatefulSet object
  // corresponding to the Alertmanager object identified by key.
  // If the object is not found, it returns a nil pointer without error.
  func (c *Operator) getStatefulSetFromAlertmanagerKey(key string) (*appsv1.StatefulSet, error) {
  	ssetName := alertmanagerKeyToStatefulSetKey(key)
  
  	obj, err := c.ssetInfs.Get(ssetName)
  	if err != nil {
  		if apierrors.IsNotFound(err) {
  			c.logger.Info("StatefulSet not found", "key", ssetName)
  			return nil, nil
  		}
  		return nil, fmt.Errorf("failed to retrieve StatefulSet from informer: %w", err)
  	}
  
  	return obj.(*appsv1.StatefulSet).DeepCopy(), nil
  }
  
  // UpdateStatus updates the status subresource of the object identified by the given
  // key.
  // UpdateStatus implements the operator.Syncer interface.
  func (c *Operator) UpdateStatus(ctx context.Context, key string) error {
  	a, err := operator.GetObjectFromKey[*monitoringv1.Alertmanager](c.alrtInfs, key)
  	if err != nil {
  		return err
  	}
  
  	if a == nil {
  		return nil
  	}
  
  	if c.rr.DeletionInProgress(a) {
  		return nil
  	}
  
  	sset, err := c.getStatefulSetFromAlertmanagerKey(key)
  	if err != nil {
  		return fmt.Errorf("failed to get StatefulSet: %w", err)
  	}
  
  	if sset != nil && c.rr.DeletionInProgress(sset) {
  		return nil
  	}
  
  	stsReporter, err := operator.NewStatefulSetReporter(ctx, c.kclient, sset)
  	if err != nil {
  		return fmt.Errorf("failed to retrieve statefulset state: %w", err)
  	}
  
  	selectorLabels := makeSelectorLabels(a.Name)
  	selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{MatchLabels: selectorLabels})
  	if err != nil {
  		return fmt.Errorf("failed to create selector for alertmanager scale status: %w", err)
  	}
  
  	a.Status.Selector = selector.String()
  	availableCondition := stsReporter.Update(a)
  	reconciledCondition := c.reconciliations.GetCondition(key, a.Generation)
  	a.Status.Conditions = operator.UpdateConditions(a.Status.Conditions, availableCondition, reconciledCondition)
  	a.Status.Paused = a.Spec.Paused
  
  	if availableCondition.Status != monitoringv1.ConditionTrue {
  		if err := stsReporter.Repair(ctx, c.logger, c.repairPolicy); err != nil {
  			c.logger.Warn("failed to repair statefulset", "err", err)
  		}
  	}
  
  	if _, err = c.mclient.MonitoringV1().Alertmanagers(a.Namespace).ApplyStatus(ctx, ApplyConfigurationFromAlertmanager(a, true), metav1.ApplyOptions{FieldManager: k8s.PrometheusOperatorFieldManager, Force: true}); err != nil {
  		c.logger.Info("failed to apply alertmanager status subresource, trying again without scale fields", "err", err)
  		// Try again, but this time does not update scale subresource.
  		if _, err = c.mclient.MonitoringV1().Alertmanagers(a.Namespace).ApplyStatus(ctx, ApplyConfigurationFromAlertmanager(a, false), metav1.ApplyOptions{FieldManager: k8s.PrometheusOperatorFieldManager, Force: true}); err != nil {
  			return fmt.Errorf("failed to apply alertmanager status subresource: %w", err)
  		}
  	}
  
  	return nil
  }
  
  // makeSelectorLabels returns the default selector for the pods of the Alertmanager statefulset.
  func makeSelectorLabels(name string) map[string]string {
  	return map[string]string{
  		operator.ApplicationNameLabelKey:     applicationNameLabelValue,
  		operator.ManagedByLabelKey:           operator.ManagedByLabelValue,
  		operator.ApplicationInstanceLabelKey: name,
  		"alertmanager":                       name,
  	}
  }
  
  // labelSelectorForStatefulSets returns a label selector which selects
  // all Alertmanager statefulsets.
  func labelSelectorForStatefulSets() string {
  	return fmt.Sprintf(
  		"%s in (%s),%s in (%s)",
  		operator.ManagedByLabelKey, operator.ManagedByLabelValue,
  		operator.ApplicationNameLabelKey, applicationNameLabelValue,
  	)
  }
  
  func createSSetInputHash(a monitoringv1.Alertmanager, c Config, tlsAssets *operator.ShardedSecret, s appsv1.StatefulSetSpec) (string, error) {
  	var http2 *bool
  	if a.Spec.Web != nil && a.Spec.Web.HTTPConfig != nil {
  		http2 = a.Spec.Web.HTTPConfig.HTTP2
  	}
  
  	// The controller should ignore any changes to RevisionHistoryLimit field because
  	// it may be modified by external actors.
  	// See https://github.com/prometheus-operator/prometheus-operator/issues/5712
  	s.RevisionHistoryLimit = nil
  
  	hash, err := hashstructure.Hash(struct {
  		AlertmanagerLabels      map[string]string
  		AlertmanagerAnnotations map[string]string
  		AlertmanagerGeneration  int64
  		AlertmanagerWebHTTP2    *bool
  		Config                  Config
  		StatefulSetSpec         appsv1.StatefulSetSpec
  		ShardedSecret           *operator.ShardedSecret
  	}{
  		AlertmanagerLabels:      a.Labels,
  		AlertmanagerAnnotations: a.Annotations,
  		AlertmanagerGeneration:  a.Generation,
  		AlertmanagerWebHTTP2:    http2,
  		Config:                  c,
  		StatefulSetSpec:         s,
  		ShardedSecret:           tlsAssets,
  	},
  		nil,
  	)
  	if err != nil {
  		return "", fmt.Errorf("failed to calculate combined hash: %w", err)
  	}
  
  	return fmt.Sprintf("%d", hash), nil
  }
  
  func defaultAlertmanagerConfiguration() []byte {
  	return []byte(`route:
    receiver: 'null'
  receivers:
  - name: 'null'`)
  }
  
  // loadConfigurationFromSecret returns the raw Alertmanager configuration and
  // additional keys from the configured secret. If the secret doesn't exist or
  // the key isn't found, it will return a working minimal data.
  func (c *Operator) loadConfigurationFromSecret(ctx context.Context, am *monitoringv1.Alertmanager) ([]byte, map[string][]byte, error) {
  	namespacedLogger := c.logger.With("alertmanager", am.Name, "namespace", am.Namespace)
  	name := defaultConfigSecretName(am)
  
  	// Tentatively retrieve the secret containing the user-provided Alertmanager
  	// configuration.
  	secret, err := c.kclient.CoreV1().Secrets(am.Namespace).Get(ctx, name, metav1.GetOptions{})
  	if err != nil {
  		if apierrors.IsNotFound(err) {
  			namespacedLogger.Info("config secret not found, using default Alertmanager configuration", "secret", name)
  			return defaultAlertmanagerConfiguration(), nil, nil
  		}
  
  		return nil, nil, err
  	}
  
  	if _, ok := secret.Data[alertmanagerConfigFile]; !ok {
  		namespacedLogger.Info("key not found in the config secret, using default Alertmanager configuration", "secret", name, "key", alertmanagerConfigFile)
  		return defaultAlertmanagerConfiguration(), secret.Data, nil
  	}
  
  	rawAlertmanagerConfig := secret.Data[alertmanagerConfigFile]
  	delete(secret.Data, alertmanagerConfigFile)
  
  	if len(rawAlertmanagerConfig) == 0 {
  		namespacedLogger.Info("empty configuration in the config secret, using default Alertmanager configuration", "secret", name, "key", alertmanagerConfigFile)
  		rawAlertmanagerConfig = defaultAlertmanagerConfiguration()
  	}
  
  	return rawAlertmanagerConfig, secret.Data, nil
  }
  
  func (c *Operator) provisionAlertmanagerConfiguration(ctx context.Context, am *monitoringv1.Alertmanager, store *assets.StoreBuilder) error {
  	amVersion := operator.StringValOrDefault(am.Spec.Version, operator.DefaultAlertmanagerVersion)
  	version, err := semver.ParseTolerant(amVersion)
  	if err != nil {
  		return fmt.Errorf("failed to parse alertmanager version: %w", err)
  	}
  
  	if version.LT(semver.MustParse("0.15.0")) || version.Major > 0 {
  		return fmt.Errorf("unsupported Alertmanager version %q", amVersion)
  	}
  
  	namespacedLogger := c.logger.With("alertmanager", am.Name, "namespace", am.Namespace)
  	// If no AlertmanagerConfig selectors and AlertmanagerConfiguration are
  	// configured, the user wants to manage configuration themselves.
  	if am.Spec.AlertmanagerConfigSelector == nil && am.Spec.AlertmanagerConfiguration == nil {
  		namespacedLogger.Debug("AlertmanagerConfigSelector and AlertmanagerConfiguration not specified, using the configuration from secret as-is",
  			"secret", defaultConfigSecretName(am))
  
  		amRawConfiguration, additionalData, err := c.loadConfigurationFromSecret(ctx, am)
  		if err != nil {
  			return fmt.Errorf("failed to retrieve configuration from secret: %w", err)
  		}
  
  		err = c.createOrUpdateGeneratedConfigSecret(ctx, am, amRawConfiguration, additionalData)
  		if err != nil {
  			return fmt.Errorf("create or update generated config secret failed: %w", err)
  		}
  
  		return nil
  	}
  
  	amConfigs, err := c.selectAlertmanagerConfigs(ctx, am, version, store)
  	if err != nil {
  		return fmt.Errorf("failed to select AlertmanagerConfig objects: %w", err)
  	}
  
  	var (
  		additionalData map[string][]byte
  		cfgBuilder     = NewConfigBuilder(namespacedLogger, version, store, am)
  	)
  
  	if am.Spec.AlertmanagerConfiguration != nil {
  		// Load the base configuration from the referenced AlertmanagerConfig.
  		globalAmConfig, err := c.mclient.MonitoringV1alpha1().AlertmanagerConfigs(am.Namespace).
  			Get(ctx, am.Spec.AlertmanagerConfiguration.Name, metav1.GetOptions{})
  		if err != nil {
  			return fmt.Errorf("failed to get global AlertmanagerConfig: %w", err)
  		}
  
  		err = cfgBuilder.initializeFromAlertmanagerConfig(ctx, am.Spec.AlertmanagerConfiguration.Global, globalAmConfig)
  		if err != nil {
  			return fmt.Errorf("failed to initialize from global AlertmanagerConfig: %w", err)
  		}
  
  		for _, v := range am.Spec.AlertmanagerConfiguration.Templates {
  			if v.ConfigMap != nil {
  				cfgBuilder.cfg.Templates = append(cfgBuilder.cfg.Templates, path.Join(alertmanagerTemplatesDir, v.ConfigMap.Key))
  			}
  			if v.Secret != nil {
  				cfgBuilder.cfg.Templates = append(cfgBuilder.cfg.Templates, path.Join(alertmanagerTemplatesDir, v.Secret.Key))
  			}
  		}
  	} else {
  		// Load the base configuration from the referenced secret.
  		var (
  			amRawConfiguration []byte
  			err                error
  		)
  
  		amRawConfiguration, additionalData, err = c.loadConfigurationFromSecret(ctx, am)
  		if err != nil {
  			return fmt.Errorf("failed to retrieve configuration from secret: %w", err)
  		}
  
  		err = cfgBuilder.InitializeFromRawConfiguration(amRawConfiguration)
  		if err != nil {
  			return fmt.Errorf("failed to initialize from secret: %w", err)
  		}
  	}
  
  	if err := cfgBuilder.AddAlertmanagerConfigs(ctx, amConfigs); err != nil {
  		return fmt.Errorf("failed to generate Alertmanager configuration: %w", err)
  	}
  
  	generatedConfig, err := cfgBuilder.MarshalJSON()
  	if err != nil {
  		return fmt.Errorf("failed to marshal configuration: %w", err)
  	}
  
  	err = c.createOrUpdateGeneratedConfigSecret(ctx, am, generatedConfig, additionalData)
  	if err != nil {
  		return fmt.Errorf("failed to create or update the generated configuration secret: %w", err)
  	}
  
  	return nil
  }
  
  func (c *Operator) createOrUpdateGeneratedConfigSecret(ctx context.Context, am *monitoringv1.Alertmanager, conf []byte, additionalData map[string][]byte) error {
  	generatedConfigSecret := &corev1.Secret{
  		Data: map[string][]byte{},
  	}
  
  	operator.UpdateObject(
  		generatedConfigSecret,
  		operator.WithLabels(c.config.Labels),
  		operator.WithAnnotations(c.config.Annotations),
  		operator.WithManagingOwner(am),
  		operator.WithName(generatedConfigSecretName(am.Name)),
  	)
  
  	maps.Copy(generatedConfigSecret.Data, additionalData)
  	// Compress config to avoid 1mb secret limit for a while
  	var buf bytes.Buffer
  	if err := operator.GzipConfig(&buf, conf); err != nil {
  		return fmt.Errorf("couldnt gzip config: %w", err)
  	}
  	generatedConfigSecret.Data[alertmanagerConfigFileCompressed] = buf.Bytes()
  
  	sClient := c.kclient.CoreV1().Secrets(am.Namespace)
  	err := k8s.CreateOrUpdateSecret(ctx, sClient, generatedConfigSecret)
  	if err != nil {
  		return fmt.Errorf("failed to update generated config secret: %w", err)
  	}
  
  	return nil
  }
  
  func (c *Operator) selectAlertmanagerConfigs(ctx context.Context, am *monitoringv1.Alertmanager, amVersion semver.Version, store *assets.StoreBuilder) (map[string]*monitoringv1alpha1.AlertmanagerConfig, error) {
  	namespaces := []string{}
  
  	// If 'AlertmanagerConfigNamespaceSelector' is nil, only check own namespace.
  	if am.Spec.AlertmanagerConfigNamespaceSelector == nil {
  		namespaces = append(namespaces, am.Namespace)
  
  		c.logger.Debug("selecting AlertmanagerConfigs from alertmanager's namespace", "namespace", am.Namespace, "alertmanager", am.Name)
  	} else {
  		amConfigNSSelector, err := metav1.LabelSelectorAsSelector(am.Spec.AlertmanagerConfigNamespaceSelector)
  		if err != nil {
  			return nil, err
  		}
  
  		err = cache.ListAll(c.nsAlrtCfgInf.GetStore(), amConfigNSSelector, func(obj any) {
  			namespaces = append(namespaces, obj.(*corev1.Namespace).Name)
  		})
  		if err != nil {
  			return nil, fmt.Errorf("failed to list namespaces: %w", err)
  		}
  
  		c.logger.Debug("filtering namespaces to select AlertmanagerConfigs from", "namespaces", strings.Join(namespaces, ","), "namespace", am.Namespace, "alertmanager", am.Name)
  	}
  
  	// Selected object might overlap, deduplicate them by `<namespace>/<name>`.
  	amConfigs := make(map[string]*monitoringv1alpha1.AlertmanagerConfig)
  
  	amConfigSelector, err := metav1.LabelSelectorAsSelector(am.Spec.AlertmanagerConfigSelector)
  	if err != nil {
  		return nil, err
  	}
  
  	for _, ns := range namespaces {
  		err := c.alrtCfgInfs.ListAllByNamespace(ns, amConfigSelector, func(obj any) {
  			k, ok := c.accessor.MetaNamespaceKey(obj)
  			if !ok {
  				return
  			}
  
  			amConfig := obj.(*monitoringv1alpha1.AlertmanagerConfig)
  			if am.Spec.AlertmanagerConfiguration != nil && amConfig.Namespace == am.Namespace && amConfig.Name == am.Spec.AlertmanagerConfiguration.Name {
  				// Skip the global AlertmanagerConfig object.
  				return
  			}
  
  			amConfigs[k] = amConfig
  		})
  		if err != nil {
  			return nil, fmt.Errorf("failed to list alertmanager configs in namespace %s: %w", ns, err)
  		}
  	}
  
  	var rejected int
  	res := make(map[string]*monitoringv1alpha1.AlertmanagerConfig, len(amConfigs))
  
  	eventRecorder := c.newEventRecorder(am)
  	for namespaceAndName, amc := range amConfigs {
  		if err := checkAlertmanagerConfigResource(ctx, amc, amVersion, store); err != nil {
  			rejected++
  			c.logger.Warn(
  				"skipping alertmanagerconfig",
  				"error", err.Error(),
  				"alertmanagerconfig", namespaceAndName,
  				"namespace", am.Namespace,
  				"alertmanager", am.Name,
  			)
  			eventRecorder.Eventf(amc, corev1.EventTypeWarning, operator.InvalidConfigurationEvent, selectingAlertmanagerConfigResourcesAction, "AlertmanagerConfig %s was rejected due to invalid configuration: %v", amc.GetName(), err)
  			continue
  		}
  
  		res[namespaceAndName] = amc
  	}
  
  	amcKeys := []string{}
  	for k := range res {
  		amcKeys = append(amcKeys, k)
  	}
  	c.logger.Debug("selected AlertmanagerConfigs", "alertmanagerconfigs", strings.Join(amcKeys, ","), "namespace", am.Namespace, "prometheus", am.Name)
  
  	if amKey, ok := c.accessor.MetaNamespaceKey(am); ok {
  		c.metrics.SetSelectedResources(amKey, monitoringv1alpha1.AlertmanagerConfigKind, len(res))
  		c.metrics.SetRejectedResources(amKey, monitoringv1alpha1.AlertmanagerConfigKind, rejected)
  	}
  
  	return res, nil
  }
  
  // checkAlertmanagerConfigResource verifies that an AlertmanagerConfig object is valid
  // for the given Alertmanager version and has no missing references to other objects.
  func checkAlertmanagerConfigResource(ctx context.Context, amc *monitoringv1alpha1.AlertmanagerConfig, amVersion semver.Version, store *assets.StoreBuilder) error {
  	// Perform semantic validation irrespective of the Alertmanager version.
  	if err := validationv1alpha1.ValidateAlertmanagerConfig(amc); err != nil {
  		return err
  	}
  
  	// Perform more specific validations which depend on the Alertmanager
  	// version. It also retrieves data from referenced secrets and configmaps
  	// (and fails in case of missing/invalid references).
  	if err := checkReceivers(ctx, amc, store, amVersion); err != nil {
  		return err
  	}
  
  	if err := checkRoute(ctx, amc.Spec.Route, amVersion); err != nil {
  		return err
  	}
  
  	return checkInhibitRules(amc, amVersion)
  }
  
  func checkRoute(ctx context.Context, route *monitoringv1alpha1.Route, amVersion semver.Version) error {
  	if route == nil {
  		return nil
  	}
  
  	matchersV2Allowed := amVersion.GTE(semver.MustParse("0.22.0"))
  	if !matchersV2Allowed && checkIsV2Matcher(route.Matchers) {
  		return fmt.Errorf(
  			`invalid syntax in route config for 'matchers' comparison based matching is supported in Alertmanager >= 0.22.0 only (matchers=%v) (receiver=%v)`,
  			route.Matchers, route.Receiver)
  	}
  
  	childRoutes, err := route.ChildRoutes()
  	if err != nil {
  		return err
  	}
  
  	for _, route := range childRoutes {
  		if err := checkRoute(ctx, &route, amVersion); err != nil {
  			return err
  		}
  	}
  
  	return nil
  }
  
  func checkHTTPConfig(hc *monitoringv1alpha1.HTTPConfig, amVersion semver.Version) error {
  	if hc == nil {
  		return nil
  	}
  
  	if hc.Authorization != nil && !amVersion.GTE(semver.MustParse("0.22.0")) {
  		return fmt.Errorf(
  			"'authorization' config set in 'httpConfig' but supported in Alertmanager >= 0.22.0 only - current %s",
  			amVersion.String(),
  		)
  	}
  
  	if hc.OAuth2 != nil && !amVersion.GTE(semver.MustParse("0.22.0")) {
  		return fmt.Errorf(
  			"'oauth2' config set in 'httpConfig' but supported in Alertmanager >= 0.22.0 only - current %s",
  			amVersion.String(),
  		)
  	}
  
  	if (hc.NoProxy != nil ||
  		hc.ProxyFromEnvironment != nil ||
  		hc.ProxyConnectHeader != nil) &&
  		amVersion.LT(semver.MustParse("0.25.0")) {
  		return fmt.Errorf(
  			"'ProxyConfig' config set in 'httpConfig' but supported in Alertmanager >= 0.25.0 only - current %s",
  			amVersion.String(),
  		)
  	}
  
  	return nil
  }
  
  func checkReceivers(ctx context.Context, amc *monitoringv1alpha1.AlertmanagerConfig, store *assets.StoreBuilder, amVersion semver.Version) error {
  	for _, receiver := range amc.Spec.Receivers {
  		err := checkPagerDutyConfigs(ctx, receiver.PagerDutyConfigs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  
  		err = checkOpsGenieConfigs(ctx, receiver.OpsGenieConfigs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  
  		err = checkDiscordConfigs(ctx, receiver.DiscordConfigs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  
  		err = checkSlackConfigs(ctx, receiver.SlackConfigs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  
  		err = checkWebhookConfigs(ctx, receiver.WebhookConfigs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  
  		err = checkWechatConfigs(ctx, receiver.WeChatConfigs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  
  		err = checkWebexConfigs(ctx, receiver.WebexConfigs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  
  		err = checkEmailConfigs(ctx, receiver.EmailConfigs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  
  		err = checkVictorOpsConfigs(ctx, receiver.VictorOpsConfigs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  
  		err = checkPushoverConfigs(ctx, receiver.PushoverConfigs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  
  		err = checkSnsConfigs(ctx, receiver.SNSConfigs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  
  		err = checkTelegramConfigs(ctx, receiver.TelegramConfigs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  
  		err = checkMSTeamsConfigs(ctx, receiver.MSTeamsConfigs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  
  		err = checkMSTeamsV2Configs(ctx, receiver.MSTeamsV2Configs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  
  		err = checkRocketChatConfigs(ctx, receiver.RocketChatConfigs, amc.GetNamespace(), store, amVersion)
  		if err != nil {
  			return err
  		}
  	}
  
  	return nil
  }
  
  func checkPagerDutyConfigs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.PagerDutyConfig,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	for _, config := range configs {
  		if err := checkHTTPConfig(config.HTTPConfig, amVersion); err != nil {
  			return err
  		}
  
  		if config.RoutingKey != nil {
  			if _, err := store.GetSecretKey(ctx, namespace, *config.RoutingKey); err != nil {
  				return err
  			}
  		}
  
  		if config.ServiceKey != nil {
  			if _, err := store.GetSecretKey(ctx, namespace, *config.ServiceKey); err != nil {
  				return err
  			}
  		}
  
  		if err := configureHTTPConfigInStore(ctx, config.HTTPConfig, namespace, store); err != nil {
  			return err
  		}
  	}
  
  	return nil
  }
  
  func checkOpsGenieConfigs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.OpsGenieConfig,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	for _, config := range configs {
  		if err := checkHTTPConfig(config.HTTPConfig, amVersion); err != nil {
  			return err
  		}
  		if err := checkOpsGenieResponder(config.Responders, amVersion); err != nil {
  			return err
  		}
  
  		if config.APIKey != nil {
  			if _, err := store.GetSecretKey(ctx, namespace, *config.APIKey); err != nil {
  				return err
  			}
  		}
  
  		if err := configureHTTPConfigInStore(ctx, config.HTTPConfig, namespace, store); err != nil {
  			return err
  		}
  	}
  
  	return nil
  }
  
  func checkOpsGenieResponder(opsgenieResponder []monitoringv1alpha1.OpsGenieConfigResponder, amVersion semver.Version) error {
  	lessThanV0_24 := amVersion.LT(semver.MustParse("0.24.0"))
  	for _, resp := range opsgenieResponder {
  		if resp.Type == "teams" && lessThanV0_24 {
  			return fmt.Errorf("'teams' set in 'opsgenieResponder' but supported in Alertmanager >= 0.24.0 only")
  		}
  	}
  	return nil
  }
  
  func checkDiscordConfigs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.DiscordConfig,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	if len(configs) == 0 {
  		return nil
  	}
  
  	if amVersion.LT(semver.MustParse("0.25.0")) {
  		return fmt.Errorf(`discordConfigs' is available in Alertmanager >= 0.25.0 only - current %s`, amVersion)
  	}
  
  	for _, config := range configs {
  		if err := checkHTTPConfig(config.HTTPConfig, amVersion); err != nil {
  			return err
  		}
  
  		if err := configureHTTPConfigInStore(ctx, config.HTTPConfig, namespace, store); err != nil {
  			return err
  		}
  
  		url, err := store.GetSecretKey(ctx, namespace, config.APIURL)
  		if err != nil {
  			return fmt.Errorf("failed to retrieve API URL: %w", err)
  		}
  		if err := validation.ValidateSecretURL(strings.TrimSpace(url)); err != nil {
  			return fmt.Errorf("failed to validate API URL: %w", err)
  		}
  	}
  
  	return nil
  }
  
  func checkRocketChatConfigs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.RocketChatConfig,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	if len(configs) == 0 {
  		return nil
  	}
  
  	if amVersion.LT(semver.MustParse("0.28.0")) {
  		return fmt.Errorf(`rocketChatConfigs' is available in Alertmanager >= 0.28.0 only - current %s`, amVersion)
  	}
  
  	for _, config := range configs {
  		if err := checkHTTPConfig(config.HTTPConfig, amVersion); err != nil {
  			return err
  		}
  
  		if err := configureHTTPConfigInStore(ctx, config.HTTPConfig, namespace, store); err != nil {
  			return err
  		}
  
  		if _, err := store.GetSecretKey(ctx, namespace, config.Token); err != nil {
  			return fmt.Errorf("failed to retrieve RocketChat token: %w", err)
  		}
  
  		if _, err := store.GetSecretKey(ctx, namespace, config.TokenID); err != nil {
  			return fmt.Errorf("failed to retrieve RocketChat token ID: %w", err)
  		}
  	}
  
  	return nil
  }
  
  func checkSlackConfigs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.SlackConfig,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	for _, config := range configs {
  		if err := checkHTTPConfig(config.HTTPConfig, amVersion); err != nil {
  			return err
  		}
  
  		if config.APIURL != nil {
  			url, err := store.GetSecretKey(ctx, namespace, *config.APIURL)
  			if err != nil {
  				return err
  			}
  			if err := validation.ValidateSecretURL(strings.TrimSpace(url)); err != nil {
  				return fmt.Errorf("failed to validate API URL: %w", err)
  			}
  		}
  
  		if config.MessageText != nil && amVersion.LT(semver.MustParse("0.31.0")) {
  			return fmt.Errorf(`messageText' is available in Alertmanager >= 0.31.0 only - current %s`, amVersion)
  		}
  
  		if err := configureHTTPConfigInStore(ctx, config.HTTPConfig, namespace, store); err != nil {
  			return err
  		}
  	}
  
  	return nil
  }
  
  func checkWebhookConfigs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.WebhookConfig,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	for _, config := range configs {
  		if err := checkHTTPConfig(config.HTTPConfig, amVersion); err != nil {
  			return err
  		}
  
  		if config.URLSecret != nil {
  			url, err := store.GetSecretKey(ctx, namespace, *config.URLSecret)
  			if err != nil {
  				return err
  			}
  
  			if err := validation.ValidateTemplateURL(strings.TrimSpace(url)); err != nil {
  				return fmt.Errorf("failed to validate URL: %w", err)
  			}
  		}
  
  		if err := configureHTTPConfigInStore(ctx, config.HTTPConfig, namespace, store); err != nil {
  			return err
  		}
  	}
  
  	return nil
  }
  
  func checkWechatConfigs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.WeChatConfig,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	for _, config := range configs {
  		if err := checkHTTPConfig(config.HTTPConfig, amVersion); err != nil {
  			return err
  		}
  
  		if config.APISecret != nil {
  			if _, err := store.GetSecretKey(ctx, namespace, *config.APISecret); err != nil {
  				return err
  			}
  		}
  
  		if err := configureHTTPConfigInStore(ctx, config.HTTPConfig, namespace, store); err != nil {
  			return err
  		}
  	}
  
  	return nil
  }
  
  func checkWebexConfigs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.WebexConfig,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	if len(configs) == 0 {
  		return nil
  	}
  
  	if amVersion.LT(semver.MustParse("0.25.0")) {
  		return fmt.Errorf(`webexConfigs' is available in Alertmanager >= 0.25.0 only - current %s`, amVersion)
  	}
  
  	for _, config := range configs {
  		if err := checkHTTPConfig(config.HTTPConfig, amVersion); err != nil {
  			return err
  		}
  
  		if err := configureHTTPConfigInStore(ctx, config.HTTPConfig, namespace, store); err != nil {
  			return err
  		}
  	}
  
  	return nil
  }
  
  func checkEmailConfigs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.EmailConfig,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	for _, config := range configs {
  		if config.AuthPassword != nil {
  			if _, err := store.GetSecretKey(ctx, namespace, *config.AuthPassword); err != nil {
  				return err
  			}
  		}
  		if config.AuthSecret != nil {
  			if _, err := store.GetSecretKey(ctx, namespace, *config.AuthSecret); err != nil {
  				return err
  			}
  		}
  
  		if err := store.AddSafeTLSConfig(ctx, namespace, config.TLSConfig); err != nil {
  			return err
  		}
  
  		if config.ForceImplicitTLS != nil && amVersion.LT(semver.MustParse("0.31.0")) {
  			return fmt.Errorf(`forceImplicitTLS' is available in Alertmanager >= 0.31.0 only - current %s`, amVersion)
  		}
  	}
  
  	return nil
  }
  
  func checkVictorOpsConfigs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.VictorOpsConfig,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	for _, config := range configs {
  		if err := checkHTTPConfig(config.HTTPConfig, amVersion); err != nil {
  			return err
  		}
  		if config.APIKey != nil {
  			if _, err := store.GetSecretKey(ctx, namespace, *config.APIKey); err != nil {
  				return err
  			}
  		}
  
  		if err := configureHTTPConfigInStore(ctx, config.HTTPConfig, namespace, store); err != nil {
  			return err
  		}
  	}
  
  	return nil
  }
  
  func checkPushoverConfigs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.PushoverConfig,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	checkSecret := func(secret *corev1.SecretKeySelector, name string) error {
  		if secret == nil {
  			return fmt.Errorf("mandatory field %s is empty", name)
  		}
  		s, err := store.GetSecretKey(ctx, namespace, *secret)
  		if err != nil {
  			return err
  		}
  		if s == "" {
  			return errors.New("mandatory field userKey is empty")
  		}
  		return nil
  	}
  
  	for _, config := range configs {
  		if err := checkHTTPConfig(config.HTTPConfig, amVersion); err != nil {
  			return err
  		}
  		if err := checkSecret(config.UserKey, "userKey"); err != nil {
  			return err
  		}
  		if err := checkSecret(config.Token, "token"); err != nil {
  			return err
  		}
  
  		if config.HTML != nil && *config.HTML && config.Monospace != nil && *config.Monospace {
  			return errors.New("html and monospace options are mutually exclusive")
  		}
  
  		if ptr.Deref(config.Expire, "") != "" {
  			if _, err := model.ParseDuration(*config.Expire); err != nil {
  				return err
  			}
  		}
  
  		if ptr.Deref(config.Retry, "") != "" {
  			if _, err := model.ParseDuration(*config.Retry); err != nil {
  				return err
  			}
  		}
  
  		if err := configureHTTPConfigInStore(ctx, config.HTTPConfig, namespace, store); err != nil {
  			return err
  		}
  	}
  
  	return nil
  }
  
  func checkSnsConfigs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.SNSConfig,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	for _, config := range configs {
  		if err := checkHTTPConfig(config.HTTPConfig, amVersion); err != nil {
  			return err
  		}
  
  		if err := store.AddSigV4(ctx, namespace, config.Sigv4); err != nil {
  			return err
  		}
  
  		if err := configureHTTPConfigInStore(ctx, config.HTTPConfig, namespace, store); err != nil {
  			return err
  		}
  	}
  	return nil
  }
  
  func checkTelegramConfigs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.TelegramConfig,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	if len(configs) == 0 {
  		return nil
  	}
  
  	if amVersion.LT(semver.MustParse("0.24.0")) {
  		return fmt.Errorf(`telegramConfigs' is available in Alertmanager >= 0.24.0 only - current %s`, amVersion)
  	}
  
  	for _, config := range configs {
  		if amVersion.LT(semver.MustParse("0.26.0")) && config.BotTokenFile != nil {
  			return fmt.Errorf(`botTokenFile' is available in Alertmanager >= 0.26.0 only - current %s`, amVersion)
  		}
  
  		if err := checkHTTPConfig(config.HTTPConfig, amVersion); err != nil {
  			return err
  		}
  
  		if config.BotToken != nil {
  			if _, err := store.GetSecretKey(ctx, namespace, *config.BotToken); err != nil {
  				return err
  			}
  		}
  
  		if err := configureHTTPConfigInStore(ctx, config.HTTPConfig, namespace, store); err != nil {
  			return err
  		}
  	}
  
  	return nil
  }
  
  func checkMSTeamsConfigs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.MSTeamsConfig,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	if len(configs) == 0 {
  		return nil
  	}
  
  	if amVersion.LT(semver.MustParse("0.26.0")) {
  		return fmt.Errorf(`invalid syntax in receivers config; msteams integration is only available in Alertmanager >= 0.26.0`)
  	}
  
  	for _, config := range configs {
  		if err := checkHTTPConfig(config.HTTPConfig, amVersion); err != nil {
  			return err
  		}
  
  		if _, err := store.GetSecretKey(ctx, namespace, config.WebhookURL); err != nil {
  			return err
  		}
  
  		if err := configureHTTPConfigInStore(ctx, config.HTTPConfig, namespace, store); err != nil {
  			return err
  		}
  	}
  
  	return nil
  }
  
  func checkMSTeamsV2Configs(
  	ctx context.Context,
  	configs []monitoringv1alpha1.MSTeamsV2Config,
  	namespace string,
  	store *assets.StoreBuilder,
  	amVersion semver.Version,
  ) error {
  	if len(configs) == 0 {
  		return nil
  	}
  
  	if amVersion.LT(semver.MustParse("0.28.0")) {
  		return fmt.Errorf(`invalid syntax in receivers config; msteamsv2 integration is only available in Alertmanager >= 0.28.0`)
  	}
  
  	for _, config := range configs {
  		if config.WebhookURL != nil {
  			if _, err := store.GetSecretKey(ctx, namespace, *config.WebhookURL); err != nil {
  				return err
  			}
  		}
  
  		if err := checkHTTPConfig(config.HTTPConfig, amVersion); err != nil {
  			return err
  		}
  
  		if err := configureHTTPConfigInStore(ctx, config.HTTPConfig, namespace, store); err != nil {
  			return err
  		}
  	}
  
  	return nil
  }
  
  func checkInhibitRules(amc *monitoringv1alpha1.AlertmanagerConfig, version semver.Version) error {
  	matchersV2Allowed := version.GTE(semver.MustParse("0.22.0"))
  
  	for i, rule := range amc.Spec.InhibitRules {
  		if !matchersV2Allowed {
  			// check if rule has provided invalid syntax and error if true
  			if checkIsV2Matcher(rule.SourceMatch, rule.TargetMatch) {
  				msg := fmt.Sprintf(
  					`'sourceMatch' and/or 'targetMatch' are using matching syntax which is supported in Alertmanager >= 0.22.0 only (sourceMatch=%v, targetMatch=%v)`,
  					rule.SourceMatch, rule.TargetMatch)
  				return errors.New(msg)
  			}
  			continue
  		}
  
  		for j, tm := range rule.TargetMatch {
  			if err := tm.Validate(); err != nil {
  				return fmt.Errorf("invalid targetMatchers[%d] in inhibitRule[%d] in config %s: %w", j, i, amc.Name, err)
  			}
  		}
  
  		for j, sm := range rule.SourceMatch {
  			if err := sm.Validate(); err != nil {
  				return fmt.Errorf("invalid sourceMatchers[%d] in inhibitRule[%d] in config %s: %w", j, i, amc.Name, err)
  			}
  		}
  	}
  
  	return nil
  }
  
  // configureHTTPConfigInStore configures the asset store for HTTPConfigs.
  func configureHTTPConfigInStore(ctx context.Context, httpConfig *monitoringv1alpha1.HTTPConfig, namespace string, store *assets.StoreBuilder) error {
  	if httpConfig == nil {
  		return nil
  	}
  
  	var err error
  	if httpConfig.BearerTokenSecret != nil {
  		if _, err = store.GetSecretKey(ctx, namespace, *httpConfig.BearerTokenSecret); err != nil {
  			return err
  		}
  	}
  
  	if err = store.AddSafeAuthorizationCredentials(ctx, namespace, httpConfig.Authorization); err != nil {
  		return err
  	}
  
  	if err = store.AddBasicAuth(ctx, namespace, httpConfig.BasicAuth); err != nil {
  		return err
  	}
  
  	if err = store.AddSafeTLSConfig(ctx, namespace, httpConfig.TLSConfig); err != nil {
  		return err
  	}
  
  	if err = store.AddProxyConfig(ctx, namespace, httpConfig.ProxyConfig); err != nil {
  		return err
  	}
  
  	return store.AddOAuth2(ctx, namespace, httpConfig.OAuth2)
  }
  
  func (c *Operator) newTLSAssetSecret(am *monitoringv1.Alertmanager) *corev1.Secret {
  	s := &corev1.Secret{
  		Data: make(map[string][]byte),
  	}
  
  	operator.UpdateObject(
  		s,
  		operator.WithLabels(c.config.Labels),
  		operator.WithAnnotations(c.config.Annotations),
  		operator.WithManagingOwner(am),
  		operator.WithName(fmt.Sprintf("%s-tls-assets", prefixedName(am.Name))),
  		operator.WithNamespace(am.Namespace),
  	)
  
  	return s
  }
  
  func (c *Operator) createOrUpdateWebConfigSecret(ctx context.Context, a *monitoringv1.Alertmanager) error {
  	var fields monitoringv1.WebConfigFileFields
  	if a.Spec.Web != nil {
  		fields = a.Spec.Web.WebConfigFileFields
  	}
  
  	webConfig, err := webconfig.New(
  		webConfigDir,
  		webConfigSecretName(a.Name),
  		fields,
  	)
  	if err != nil {
  		return fmt.Errorf("failed to initialize web config: %w", err)
  	}
  
  	s := &corev1.Secret{}
  	operator.UpdateObject(
  		s,
  		operator.WithLabels(c.config.Labels),
  		operator.WithAnnotations(c.config.Annotations),
  		operator.WithManagingOwner(a),
  	)
  
  	if err := webConfig.CreateOrUpdateWebConfigSecret(ctx, c.kclient.CoreV1().Secrets(a.Namespace), s); err != nil {
  		return fmt.Errorf("failed to reconcile web config secret: %w", err)
  	}
  
  	return nil
  }
  
  func (c *Operator) createOrUpdateClusterTLSConfigSecret(ctx context.Context, a *monitoringv1.Alertmanager) error {
  	clusterTLSConfig, err := clustertlsconfig.New(clusterTLSConfigDir, a)
  	if err != nil {
  		return fmt.Errorf("failed to initialize the configuration: %w", err)
  	}
  
  	data, err := clusterTLSConfig.ClusterTLSConfiguration()
  	if err != nil {
  		return fmt.Errorf("failed to generate the configuration: %w", err)
  	}
  
  	s := &corev1.Secret{
  		ObjectMeta: metav1.ObjectMeta{
  			Name: clusterTLSConfig.GetSecretName(),
  		},
  		Data: map[string][]byte{
  			clustertlsconfig.ConfigFileKey: data,
  		},
  	}
  	operator.UpdateObject(
  		s,
  		operator.WithLabels(c.config.Labels),
  		operator.WithAnnotations(c.config.Annotations),
  		operator.WithManagingOwner(a),
  	)
  
  	if err = k8s.CreateOrUpdateSecret(ctx, c.kclient.CoreV1().Secrets(a.Namespace), s); err != nil {
  		return fmt.Errorf("failed to reconcile secret: %w", err)
  	}
  
  	return nil
  }
  
  func (c *Operator) recordDeprecatedFields(key string, logger *slog.Logger, a *monitoringv1.Alertmanager) {
  	deprecationWarningf := "field %q is deprecated, field %q should be used instead"
  	var deprecations []string
  
  	if a.Spec.BaseImage != "" {
  		deprecations = append(deprecations, fmt.Sprintf(deprecationWarningf, "spec.baseImage", "spec.image"))
  	}
  
  	if a.Spec.Tag != "" {
  		deprecations = append(deprecations, fmt.Sprintf(deprecationWarningf, "spec.tag", "spec.image"))
  	}
  
  	if a.Spec.SHA != "" {
  		deprecations = append(deprecations, fmt.Sprintf(deprecationWarningf, "spec.sha", "spec.image"))
  	}
  
  	if len(deprecations) > 0 {
  		for _, m := range deprecations {
  			logger.Warn(m)
  		}
  		c.reconciliations.SetReasonAndMessage(key, operator.DeprecatedFieldsInUseReason, strings.Join(deprecations, "; "))
  	}
  }
  
  func ApplyConfigurationFromAlertmanager(a *monitoringv1.Alertmanager, updateScaleSubresource bool) *monitoringv1ac.AlertmanagerApplyConfiguration {
  	asac := monitoringv1ac.AlertmanagerStatus().
  		WithPaused(a.Status.Paused).
  		WithReplicas(a.Status.Replicas).
  		WithAvailableReplicas(a.Status.AvailableReplicas).
  		WithUpdatedReplicas(a.Status.UpdatedReplicas).
  		WithUnavailableReplicas(a.Status.UnavailableReplicas)
  
  	if updateScaleSubresource {
  		asac = asac.WithSelector(a.Status.Selector)
  	}
  
  	for _, condition := range a.Status.Conditions {
  		asac.WithConditions(
  			monitoringv1ac.Condition().
  				WithType(condition.Type).
  				WithStatus(condition.Status).
  				WithLastTransitionTime(condition.LastTransitionTime).
  				WithReason(condition.Reason).
  				WithMessage(condition.Message).
  				WithObservedGeneration(condition.ObservedGeneration),
  		)
  	}
  
  	return monitoringv1ac.Alertmanager(a.Name, a.Namespace).WithStatus(asac)
  }
  