<commit_before>/*
 * Copyright (C) 2013 Matthew Dawson <matthew@mjdsystems.ca>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package main

import (
	"net/url"

	riak "github.com/tpjg/goriakpbc"

	"testing"
)

func makeIdGenerator() <-chan uint64 {
	ch := make(chan uint64)

	go func(ch chan<- uint64) {
		var id uint64
		for {
			ch <- id
			id++
		}
	}(ch)

	return ch
}

var testIdGenerator <-chan uint64 = makeIdGenerator()

func fixFeedForMerging(toFix *ParsedFeedData) *ParsedFeedData {
	// Instead of sorting, verify we are sorted.  Otherwise items without publication dates can end
	// up in the wrong place.
	if !toFix.Items.VerifySort() {
		panic("Items are not sorted!!!!")
	}

	for i, _ := range toFix.Items {
		item := &toFix.Items[i]
		item.GenericKey = makeHash(string(item.GenericKey))
	}
	return toFix
}

func compareParsedToFinalFeed(t *testing.T, data *ParsedFeedData, model *Feed, con *riak.Client) bool {
	// Compare basics:
	if data.Title != model.Title {
		t.Error("Feed title didn't match %s vs %s!", data.Title, model.Title)
		return false
	}

	if len(data.Items) != len(model.ItemKeys) {
		if len(data.Items) > MaximumFeedItems {
			t.Errorf("Item count differs due to items count greater then Maximum number of feed items (%v of %v)", len(data.Items), MaximumFeedItems)
		} else {
			t.Errorf("Item count is different %v vs %v!", len(data.Items), len(model.ItemKeys))
		}
		return false
	}
	if len(model.InsertedItemKeys) != 0 || len(model.DeletedItemKeys) != 0 {
		t.Error("There are left over inserted or deleted item keys!")
		return false
	}

	//Compare saved feed items.  This means a trip through riak!  The order should match though ...
	for i, itemKey := range model.ItemKeys {
		modelItem := FeedItem{}
		if err := con.LoadModel(string(itemKey), &modelItem); err != nil {
			t.Errorf("Failed to load item! Error: %s", err)
			return false
		}

		dataItem := data.Items[i]

		if dataItem.Title != modelItem.Title ||
			dataItem.Author != modelItem.Author ||
			dataItem.Content != modelItem.Content ||
			dataItem.Url != modelItem.Url ||
			!dataItem.PubDate.Equal(modelItem.PubDate) {
			t.Errorf("Item data didn't match! Original:\n%#v\nLoaded:\n%#v", dataItem, modelItem)
			return false
		}
	}

	return true
}

func CreateFeed(t *testing.T, con *riak.Client, Url *url.URL) Feed {
	feedModel := &Feed{Url: *Url}
	if err := con.NewModel(feedModel.UrlKey(), feedModel); err != nil {
		t.Fatalf("Failed to initialize feed model (%s)!", err)
	} else if err = feedModel.Save(); err != nil {
		t.Fatalf("Failed to store feed model (%s)!", err)
	}
	return *feedModel
}

func MustUpdateFeedTo(t *testing.T, con *riak.Client, url *url.URL, feedName string, updateCount int) (feed *ParsedFeedData) {
	for i := 0; i < updateCount; i++ {
		feed = getFeedDataFor(t, feedName, i)
		fixFeedForMerging(feed)

		if err := updateFeed(con, *url, *feed, testIdGenerator); err != nil {
			t.Errorf("Failed to update simple single feed (%s)!", err)
		}
	}

	return
}

func TestSingleFeedInsert(t *testing.T) {
	con := getTestConnection(t)
	defer killTestDb(con, t)

	feed := getFeedDataFor(t, "simple", 0)
	fixFeedForMerging(feed)

	url, _ := url.Parse("http://example.com/rss")

	err := updateFeed(con, *url, *feed, testIdGenerator)
	if err != FeedNotFound {
		t.Fatalf("Failed to insert simple single feed (%s)!", err)
	}

	feedModel := CreateFeed(t, con, url)

	err = updateFeed(con, *url, *feed, testIdGenerator)
	if err != nil {
		t.Errorf("Failed to update simple single feed (%s)!", err)
	}

	// Finally, load the feed again and verify properties!
	loadFeed := &Feed{}
	if err = con.LoadModel(feedModel.UrlKey(), loadFeed); err != nil {
		t.Fatalf("Failed to initialize feed model (%s)!", err)
	} else if compareParsedToFinalFeed(t, feed, loadFeed, con) != true {
		t.Errorf("Saved feed does not match what was inserted! Original:\n%+v\nLoaded:\n%+v", feed, loadFeed)
	}
}

func TestSingleFeedUpdate(t *testing.T) {
	con := getTestConnection(t)
	defer killTestDb(con, t)

	url, _ := url.Parse("http://example.com/rss")

	feedModel := CreateFeed(t, con, url)

	feed := MustUpdateFeedTo(t, con, url, "simple", 2)

	// Finally, load the feed again and verify properties!
	loadFeed := &Feed{}
	if err := con.LoadModel(feedModel.UrlKey(), loadFeed); err != nil {
		t.Fatalf("Failed to initialize feed model (%s)!", err)
	} else if compareParsedToFinalFeed(t, feed, loadFeed, con) != true {
		t.Errorf("Saved feed does not match what was inserted! Original:\n%+v\nLoaded:\n%+v", feed, loadFeed)
	}
}

func TestFeedUpdateAndInsert(t *testing.T) {
	con := getTestConnection(t)
	defer killTestDb(con, t)

	url, _ := url.Parse("http://example.com/rss")

	feedModel := CreateFeed(t, con, url)

	feed := MustUpdateFeedTo(t, con, url, "simple", 3)

	// Finally, load the feed again and verify properties!
	loadFeed := &Feed{}
	if err := con.LoadModel(feedModel.UrlKey(), loadFeed); err != nil {
		t.Fatalf("Failed to initialize feed model (%s)!", err)
	} else if compareParsedToFinalFeed(t, feed, loadFeed, con) != true {
		t.Errorf("Saved feed does not match what was inserted! Original:\n%+v\nLoaded:\n%+v", feed, loadFeed)
	}
}

func TestFeedUpdateWithChangingPubDates(t *testing.T) {
	con := getTestConnection(t)
	defer killTestDb(con, t)

	url, _ := url.Parse("http://example.com/rss")

	feedModel := CreateFeed(t, con, url)

	feed := MustUpdateFeedTo(t, con, url, "simple", 4)

	// Finally, load the feed again and verify properties!
	loadFeed := &Feed{}
	if err := con.LoadModel(feedModel.UrlKey(), loadFeed); err != nil {
		t.Fatalf("Failed to initialize feed model (%s)!", err)
	} else if compareParsedToFinalFeed(t, feed, loadFeed, con) != true {
		t.Errorf("Saved feed does not match what was inserted! Original:\n%+v\nLoaded:\n%+v", feed, loadFeed)
	}
}
<commit_msg>Fix the display of the title mismatch error for feeds.<commit_after>/*
 * Copyright (C) 2013 Matthew Dawson <matthew@mjdsystems.ca>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package main

import (
	"net/url"

	riak "github.com/tpjg/goriakpbc"

	"testing"
)

func makeIdGenerator() <-chan uint64 {
	ch := make(chan uint64)

	go func(ch chan<- uint64) {
		var id uint64
		for {
			ch <- id
			id++
		}
	}(ch)

	return ch
}

var testIdGenerator <-chan uint64 = makeIdGenerator()

func fixFeedForMerging(toFix *ParsedFeedData) *ParsedFeedData {
	// Instead of sorting, verify we are sorted.  Otherwise items without publication dates can end
	// up in the wrong place.
	if !toFix.Items.VerifySort() {
		panic("Items are not sorted!!!!")
	}

	for i, _ := range toFix.Items {
		item := &toFix.Items[i]
		item.GenericKey = makeHash(string(item.GenericKey))
	}
	return toFix
}

func compareParsedToFinalFeed(t *testing.T, data *ParsedFeedData, model *Feed, con *riak.Client) bool {
	// Compare basics:
	if data.Title != model.Title {
		t.Errorf("Feed title didn't match '%s' vs '%s'!", data.Title, model.Title)
		return false
	}

	if len(data.Items) != len(model.ItemKeys) {
		if len(data.Items) > MaximumFeedItems {
			t.Errorf("Item count differs due to items count greater then Maximum number of feed items (%v of %v)", len(data.Items), MaximumFeedItems)
		} else {
			t.Errorf("Item count is different %v vs %v!", len(data.Items), len(model.ItemKeys))
		}
		return false
	}
	if len(model.InsertedItemKeys) != 0 || len(model.DeletedItemKeys) != 0 {
		t.Error("There are left over inserted or deleted item keys!")
		return false
	}

	//Compare saved feed items.  This means a trip through riak!  The order should match though ...
	for i, itemKey := range model.ItemKeys {
		modelItem := FeedItem{}
		if err := con.LoadModel(string(itemKey), &modelItem); err != nil {
			t.Errorf("Failed to load item! Error: %s", err)
			return false
		}

		dataItem := data.Items[i]

		if dataItem.Title != modelItem.Title ||
			dataItem.Author != modelItem.Author ||
			dataItem.Content != modelItem.Content ||
			dataItem.Url != modelItem.Url ||
			!dataItem.PubDate.Equal(modelItem.PubDate) {
			t.Errorf("Item data didn't match! Original:\n%#v\nLoaded:\n%#v", dataItem, modelItem)
			return false
		}
	}

	return true
}

func CreateFeed(t *testing.T, con *riak.Client, Url *url.URL) Feed {
	feedModel := &Feed{Url: *Url}
	if err := con.NewModel(feedModel.UrlKey(), feedModel); err != nil {
		t.Fatalf("Failed to initialize feed model (%s)!", err)
	} else if err = feedModel.Save(); err != nil {
		t.Fatalf("Failed to store feed model (%s)!", err)
	}
	return *feedModel
}

func MustUpdateFeedTo(t *testing.T, con *riak.Client, url *url.URL, feedName string, updateCount int) (feed *ParsedFeedData) {
	for i := 0; i < updateCount; i++ {
		feed = getFeedDataFor(t, feedName, i)
		fixFeedForMerging(feed)

		if err := updateFeed(con, *url, *feed, testIdGenerator); err != nil {
			t.Errorf("Failed to update simple single feed (%s)!", err)
		}
	}

	return
}

func TestSingleFeedInsert(t *testing.T) {
	con := getTestConnection(t)
	defer killTestDb(con, t)

	feed := getFeedDataFor(t, "simple", 0)
	fixFeedForMerging(feed)

	url, _ := url.Parse("http://example.com/rss")

	err := updateFeed(con, *url, *feed, testIdGenerator)
	if err != FeedNotFound {
		t.Fatalf("Failed to insert simple single feed (%s)!", err)
	}

	feedModel := CreateFeed(t, con, url)

	err = updateFeed(con, *url, *feed, testIdGenerator)
	if err != nil {
		t.Errorf("Failed to update simple single feed (%s)!", err)
	}

	// Finally, load the feed again and verify properties!
	loadFeed := &Feed{}
	if err = con.LoadModel(feedModel.UrlKey(), loadFeed); err != nil {
		t.Fatalf("Failed to initialize feed model (%s)!", err)
	} else if compareParsedToFinalFeed(t, feed, loadFeed, con) != true {
		t.Errorf("Saved feed does not match what was inserted! Original:\n%+v\nLoaded:\n%+v", feed, loadFeed)
	}
}

func TestSingleFeedUpdate(t *testing.T) {
	con := getTestConnection(t)
	defer killTestDb(con, t)

	url, _ := url.Parse("http://example.com/rss")

	feedModel := CreateFeed(t, con, url)

	feed := MustUpdateFeedTo(t, con, url, "simple", 2)

	// Finally, load the feed again and verify properties!
	loadFeed := &Feed{}
	if err := con.LoadModel(feedModel.UrlKey(), loadFeed); err != nil {
		t.Fatalf("Failed to initialize feed model (%s)!", err)
	} else if compareParsedToFinalFeed(t, feed, loadFeed, con) != true {
		t.Errorf("Saved feed does not match what was inserted! Original:\n%+v\nLoaded:\n%+v", feed, loadFeed)
	}
}

func TestFeedUpdateAndInsert(t *testing.T) {
	con := getTestConnection(t)
	defer killTestDb(con, t)

	url, _ := url.Parse("http://example.com/rss")

	feedModel := CreateFeed(t, con, url)

	feed := MustUpdateFeedTo(t, con, url, "simple", 3)

	// Finally, load the feed again and verify properties!
	loadFeed := &Feed{}
	if err := con.LoadModel(feedModel.UrlKey(), loadFeed); err != nil {
		t.Fatalf("Failed to initialize feed model (%s)!", err)
	} else if compareParsedToFinalFeed(t, feed, loadFeed, con) != true {
		t.Errorf("Saved feed does not match what was inserted! Original:\n%+v\nLoaded:\n%+v", feed, loadFeed)
	}
}

func TestFeedUpdateWithChangingPubDates(t *testing.T) {
	con := getTestConnection(t)
	defer killTestDb(con, t)

	url, _ := url.Parse("http://example.com/rss")

	feedModel := CreateFeed(t, con, url)

	feed := MustUpdateFeedTo(t, con, url, "simple", 4)

	// Finally, load the feed again and verify properties!
	loadFeed := &Feed{}
	if err := con.LoadModel(feedModel.UrlKey(), loadFeed); err != nil {
		t.Fatalf("Failed to initialize feed model (%s)!", err)
	} else if compareParsedToFinalFeed(t, feed, loadFeed, con) != true {
		t.Errorf("Saved feed does not match what was inserted! Original:\n%+v\nLoaded:\n%+v", feed, loadFeed)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package k8sclient

import (
	"fmt"
	"strings"
	"time"

	autoscalingv1 "k8s.io/api/autoscaling/v1"
	"k8s.io/api/core/v1"
	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"

	"github.com/golang/glog"
)

// K8sClient - Wraps all needed client functionalities for autoscaler
type K8sClient interface {
	// FetchConfigMap fetches the requested configmap from the Apiserver
	FetchConfigMap(namespace, configmap string) (*v1.ConfigMap, error)
	// CreateConfigMap creates a configmap with given namespace, name and params
	CreateConfigMap(namespace, configmap string, params map[string]string) (*v1.ConfigMap, error)
	// UpdateConfigMap updates a configmap with given namespace, name and params
	UpdateConfigMap(namespace, configmap string, params map[string]string) (*v1.ConfigMap, error)
	// GetClusterStatus counts schedulable nodes and cores in the cluster
	GetClusterStatus() (clusterStatus *ClusterStatus, err error)
	// GetNamespace returns the namespace of target resource.
	GetNamespace() (namespace string)
	// UpdateReplicas updates the number of replicas for the resource and return the previous replicas count
	UpdateReplicas(expReplicas int32) (prevReplicas int32, err error)
}

// k8sClient - Wraps all Kubernetes API client functionalities
type k8sClient struct {
	target        *scaleTarget
	clientset     *kubernetes.Clientset
	clusterStatus *ClusterStatus
	nodeStore     cache.Store
	reflector     *cache.Reflector
	stopCh        chan struct{}
}

// NewK8sClient gives a k8sClient with the given dependencies.
func NewK8sClient(namespace, target string, nodelabels string) (K8sClient, error) {
	config, err := rest.InClusterConfig()
	if err != nil {
		return nil, err
	}
	// Use protobufs for communication with apiserver.
	config.ContentType = "application/vnd.kubernetes.protobuf"
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, err
	}

	scaleTarget, err := getScaleTarget(target, namespace)
	if err != nil {
		return nil, err
	}

	// Start propagating contents of the nodeStore.

	opts := metav1.ListOptions{LabelSelector: nodelabels}
	nodeListWatch := &cache.ListWatch{
		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
			return clientset.CoreV1().Nodes().List(opts)
		},
		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
			return clientset.CoreV1().Nodes().Watch(opts)
		},
	}
	nodeStore := cache.NewStore(cache.MetaNamespaceKeyFunc)
	reflector := cache.NewReflector(nodeListWatch, &v1.Node{}, nodeStore, 0)
	stopCh := make(chan struct{})
	go reflector.Run(stopCh)

	return &k8sClient{
		target:    scaleTarget,
		clientset: clientset,
		nodeStore: nodeStore,
		reflector: reflector,
		stopCh:    stopCh,
	}, nil
}

func getScaleTarget(target, namespace string) (*scaleTarget, error) {
	splits := strings.Split(target, "/")
	if len(splits) != 2 {
		return &scaleTarget{}, fmt.Errorf("target format error: %v", target)
	}
	kind := splits[0]
	name := splits[1]
	return &scaleTarget{kind, name, namespace}, nil
}

// scaleTarget stores the scalable target recourse
type scaleTarget struct {
	kind      string
	name      string
	namespace string
}

func (k *k8sClient) GetNamespace() (namespace string) {
	return k.target.namespace
}

func (k *k8sClient) FetchConfigMap(namespace, configmap string) (*v1.ConfigMap, error) {
	cm, err := k.clientset.CoreV1().ConfigMaps(namespace).Get(configmap, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}
	return cm, nil
}

func (k *k8sClient) CreateConfigMap(namespace, configmap string, params map[string]string) (*v1.ConfigMap, error) {
	providedConfigMap := v1.ConfigMap{}
	providedConfigMap.ObjectMeta.Name = configmap
	providedConfigMap.ObjectMeta.Namespace = namespace
	providedConfigMap.Data = params
	cm, err := k.clientset.CoreV1().ConfigMaps(namespace).Create(&providedConfigMap)
	if err != nil {
		return nil, err
	}
	glog.V(0).Infof("Created ConfigMap %v in namespace %v", configmap, namespace)
	return cm, nil
}

func (k *k8sClient) UpdateConfigMap(namespace, configmap string, params map[string]string) (*v1.ConfigMap, error) {
	providedConfigMap := v1.ConfigMap{}
	providedConfigMap.ObjectMeta.Name = configmap
	providedConfigMap.ObjectMeta.Namespace = namespace
	providedConfigMap.Data = params
	cm, err := k.clientset.CoreV1().ConfigMaps(namespace).Update(&providedConfigMap)
	if err != nil {
		return nil, err
	}
	glog.V(0).Infof("Updated ConfigMap %v in namespace %v", configmap, namespace)
	return cm, nil
}

// ClusterStatus defines the cluster status
type ClusterStatus struct {
	TotalNodes       int32
	SchedulableNodes int32
	TotalCores       int32
	SchedulableCores int32
}

func (k *k8sClient) GetClusterStatus() (clusterStatus *ClusterStatus, err error) {
	// TODO: Consider moving this to NewK8sClient method and failing fast when
	// reflector can't initialize. That is a tradeoff between silently non-working
	// component and explicit restarts of it. In majority of the cases the restart
	// won't repair it - though it may give better visibility into problems.
	err = wait.PollImmediate(250*time.Millisecond, 5*time.Second, func() (bool, error) {
		if k.reflector.LastSyncResourceVersion() == "" {
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		return nil, err
	}
	nodes := k.nodeStore.List()

	clusterStatus = &ClusterStatus{}
	clusterStatus.TotalNodes = int32(len(nodes))
	var tc resource.Quantity
	var sc resource.Quantity
	for i := range nodes {
		node, ok := nodes[i].(*v1.Node)
		if !ok {
			glog.Errorf("Unexpected object: %#v", nodes[i])
			continue
		}
		tc.Add(node.Status.Capacity[v1.ResourceCPU])
		if !node.Spec.Unschedulable {
			clusterStatus.SchedulableNodes++
			sc.Add(node.Status.Capacity[v1.ResourceCPU])
		}
	}

	tcInt64, tcOk := tc.AsInt64()
	scInt64, scOk := sc.AsInt64()
	if !tcOk || !scOk {
		return nil, fmt.Errorf("unable to compute integer values of schedulable cores in the cluster")
	}
	clusterStatus.TotalCores = int32(tcInt64)
	clusterStatus.SchedulableCores = int32(scInt64)
	k.clusterStatus = clusterStatus
	return clusterStatus, nil
}

func (k *k8sClient) UpdateReplicas(expReplicas int32) (prevRelicas int32, err error) {
	prevRelicas, err = k.updateReplicasAppsV1(expReplicas)
	if err == nil || !apierrors.IsForbidden(err) {
		return prevRelicas, err
	}
	glog.V(1).Infof("Falling back to extensions/v1beta1, error using apps/v1: %v", err)

	// Fall back to using the extensions API if we get a forbidden error
	scale, err := k.getScaleExtensionsV1beta1(k.target)
	if err != nil {
		return 0, err
	}
	prevRelicas = scale.Spec.Replicas
	if expReplicas != prevRelicas {
		glog.V(0).Infof("Cluster status: SchedulableNodes[%v], SchedulableCores[%v]", k.clusterStatus.SchedulableNodes, k.clusterStatus.SchedulableCores)
		glog.V(0).Infof("Replicas are not as expected : updating replicas from %d to %d", prevRelicas, expReplicas)
		scale.Spec.Replicas = expReplicas
		_, err = k.updateScaleExtensionsV1beta1(k.target, scale)
		if err != nil {
			return 0, err
		}
	}
	return prevRelicas, nil
}

func (k *k8sClient) getScaleExtensionsV1beta1(target *scaleTarget) (*extensionsv1beta1.Scale, error) {
	opt := metav1.GetOptions{}
	switch strings.ToLower(target.kind) {
	case "deployment", "deployments":
		return k.clientset.ExtensionsV1beta1().Deployments(target.namespace).GetScale(target.name, opt)
	case "replicaset", "replicasets":
		return k.clientset.ExtensionsV1beta1().ReplicaSets(target.namespace).GetScale(target.name, opt)
	default:
		return nil, fmt.Errorf("unsupported target kind: %v", target.kind)
	}
}

func (k *k8sClient) updateScaleExtensionsV1beta1(target *scaleTarget, scale *extensionsv1beta1.Scale) (*extensionsv1beta1.Scale, error) {
	switch strings.ToLower(target.kind) {
	case "deployment", "deployments":
		return k.clientset.ExtensionsV1beta1().Deployments(target.namespace).UpdateScale(target.name, scale)
	case "replicaset", "replicasets":
		return k.clientset.ExtensionsV1beta1().ReplicaSets(target.namespace).UpdateScale(target.name, scale)
	default:
		return nil, fmt.Errorf("unsupported target kind: %v", target.kind)
	}
}

func (k *k8sClient) updateReplicasAppsV1(expReplicas int32) (prevRelicas int32, err error) {
	req, err := requestForTarget(k.clientset.AppsV1().RESTClient().Get(), k.target)
	if err != nil {
		return 0, err
	}

	scale := &autoscalingv1.Scale{}
	if err = req.Do().Into(scale); err != nil {
		return 0, err
	}

	prevRelicas = scale.Spec.Replicas
	if expReplicas != prevRelicas {
		glog.V(0).Infof("Cluster status: SchedulableNodes[%v], SchedulableCores[%v]", k.clusterStatus.SchedulableNodes, k.clusterStatus.SchedulableCores)
		glog.V(0).Infof("Replicas are not as expected : updating replicas from %d to %d", prevRelicas, expReplicas)
		scale.Spec.Replicas = expReplicas
		req, err = requestForTarget(k.clientset.AppsV1().RESTClient().Put(), k.target)
		if err != nil {
			return 0, err
		}
		if err = req.Body(scale).Do().Error(); err != nil {
			return 0, err
		}
	}

	return prevRelicas, nil
}

func requestForTarget(req *rest.Request, target *scaleTarget) (*rest.Request, error) {
	var absPath, resource string
	// Support the kinds we allowed scaling via the extensions API group
	// TODO: switch to use the polymorphic scale client once client-go versions are updated
	switch strings.ToLower(target.kind) {
	case "deployment", "deployments":
		absPath = "/apis/apps/v1"
		resource = "deployments"
	case "replicaset", "replicasets":
		absPath = "/apis/apps/v1"
		resource = "replicasets"
	case "statefulset", "statefulsets":
		absPath = "/apis/apps/v1"
		resource = "statefulsets"
	case "replicationcontroller", "replicationcontrollers":
		absPath = "/api/v1"
		resource = "replicationcontrollers"
	default:
		return nil, fmt.Errorf("unsupported target kind: %v", target.kind)
	}

	return req.AbsPath(absPath).Namespace(target.namespace).Resource(resource).Name(target.name).SubResource("scale"), nil
}
<commit_msg>use node.Status.Allocatable when calculate cores<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package k8sclient

import (
	"fmt"
	"strings"
	"time"

	autoscalingv1 "k8s.io/api/autoscaling/v1"
	"k8s.io/api/core/v1"
	extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"

	"github.com/golang/glog"
)

// K8sClient - Wraps all needed client functionalities for autoscaler
type K8sClient interface {
	// FetchConfigMap fetches the requested configmap from the Apiserver
	FetchConfigMap(namespace, configmap string) (*v1.ConfigMap, error)
	// CreateConfigMap creates a configmap with given namespace, name and params
	CreateConfigMap(namespace, configmap string, params map[string]string) (*v1.ConfigMap, error)
	// UpdateConfigMap updates a configmap with given namespace, name and params
	UpdateConfigMap(namespace, configmap string, params map[string]string) (*v1.ConfigMap, error)
	// GetClusterStatus counts schedulable nodes and cores in the cluster
	GetClusterStatus() (clusterStatus *ClusterStatus, err error)
	// GetNamespace returns the namespace of target resource.
	GetNamespace() (namespace string)
	// UpdateReplicas updates the number of replicas for the resource and return the previous replicas count
	UpdateReplicas(expReplicas int32) (prevReplicas int32, err error)
}

// k8sClient - Wraps all Kubernetes API client functionalities
type k8sClient struct {
	target        *scaleTarget
	clientset     *kubernetes.Clientset
	clusterStatus *ClusterStatus
	nodeStore     cache.Store
	reflector     *cache.Reflector
	stopCh        chan struct{}
}

// NewK8sClient gives a k8sClient with the given dependencies.
func NewK8sClient(namespace, target string, nodelabels string) (K8sClient, error) {
	config, err := rest.InClusterConfig()
	if err != nil {
		return nil, err
	}
	// Use protobufs for communication with apiserver.
	config.ContentType = "application/vnd.kubernetes.protobuf"
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, err
	}

	scaleTarget, err := getScaleTarget(target, namespace)
	if err != nil {
		return nil, err
	}

	// Start propagating contents of the nodeStore.

	opts := metav1.ListOptions{LabelSelector: nodelabels}
	nodeListWatch := &cache.ListWatch{
		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
			return clientset.CoreV1().Nodes().List(opts)
		},
		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
			return clientset.CoreV1().Nodes().Watch(opts)
		},
	}
	nodeStore := cache.NewStore(cache.MetaNamespaceKeyFunc)
	reflector := cache.NewReflector(nodeListWatch, &v1.Node{}, nodeStore, 0)
	stopCh := make(chan struct{})
	go reflector.Run(stopCh)

	return &k8sClient{
		target:    scaleTarget,
		clientset: clientset,
		nodeStore: nodeStore,
		reflector: reflector,
		stopCh:    stopCh,
	}, nil
}

func getScaleTarget(target, namespace string) (*scaleTarget, error) {
	splits := strings.Split(target, "/")
	if len(splits) != 2 {
		return &scaleTarget{}, fmt.Errorf("target format error: %v", target)
	}
	kind := splits[0]
	name := splits[1]
	return &scaleTarget{kind, name, namespace}, nil
}

// scaleTarget stores the scalable target recourse
type scaleTarget struct {
	kind      string
	name      string
	namespace string
}

func (k *k8sClient) GetNamespace() (namespace string) {
	return k.target.namespace
}

func (k *k8sClient) FetchConfigMap(namespace, configmap string) (*v1.ConfigMap, error) {
	cm, err := k.clientset.CoreV1().ConfigMaps(namespace).Get(configmap, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}
	return cm, nil
}

func (k *k8sClient) CreateConfigMap(namespace, configmap string, params map[string]string) (*v1.ConfigMap, error) {
	providedConfigMap := v1.ConfigMap{}
	providedConfigMap.ObjectMeta.Name = configmap
	providedConfigMap.ObjectMeta.Namespace = namespace
	providedConfigMap.Data = params
	cm, err := k.clientset.CoreV1().ConfigMaps(namespace).Create(&providedConfigMap)
	if err != nil {
		return nil, err
	}
	glog.V(0).Infof("Created ConfigMap %v in namespace %v", configmap, namespace)
	return cm, nil
}

func (k *k8sClient) UpdateConfigMap(namespace, configmap string, params map[string]string) (*v1.ConfigMap, error) {
	providedConfigMap := v1.ConfigMap{}
	providedConfigMap.ObjectMeta.Name = configmap
	providedConfigMap.ObjectMeta.Namespace = namespace
	providedConfigMap.Data = params
	cm, err := k.clientset.CoreV1().ConfigMaps(namespace).Update(&providedConfigMap)
	if err != nil {
		return nil, err
	}
	glog.V(0).Infof("Updated ConfigMap %v in namespace %v", configmap, namespace)
	return cm, nil
}

// ClusterStatus defines the cluster status
type ClusterStatus struct {
	TotalNodes       int32
	SchedulableNodes int32
	TotalCores       int32
	SchedulableCores int32
}

func (k *k8sClient) GetClusterStatus() (clusterStatus *ClusterStatus, err error) {
	// TODO: Consider moving this to NewK8sClient method and failing fast when
	// reflector can't initialize. That is a tradeoff between silently non-working
	// component and explicit restarts of it. In majority of the cases the restart
	// won't repair it - though it may give better visibility into problems.
	err = wait.PollImmediate(250*time.Millisecond, 5*time.Second, func() (bool, error) {
		if k.reflector.LastSyncResourceVersion() == "" {
			return false, nil
		}
		return true, nil
	})
	if err != nil {
		return nil, err
	}
	nodes := k.nodeStore.List()

	clusterStatus = &ClusterStatus{}
	clusterStatus.TotalNodes = int32(len(nodes))
	var tc resource.Quantity
	var sc resource.Quantity
	for i := range nodes {
		node, ok := nodes[i].(*v1.Node)
		if !ok {
			glog.Errorf("Unexpected object: %#v", nodes[i])
			continue
		}
		tc.Add(node.Status.Allocatable[v1.ResourceCPU])
		if !node.Spec.Unschedulable {
			clusterStatus.SchedulableNodes++
			sc.Add(node.Status.Allocatable[v1.ResourceCPU])
		}
	}

	tcInt64, tcOk := tc.AsInt64()
	scInt64, scOk := sc.AsInt64()
	if !tcOk || !scOk {
		return nil, fmt.Errorf("unable to compute integer values of schedulable cores in the cluster")
	}
	clusterStatus.TotalCores = int32(tcInt64)
	clusterStatus.SchedulableCores = int32(scInt64)
	k.clusterStatus = clusterStatus
	return clusterStatus, nil
}

func (k *k8sClient) UpdateReplicas(expReplicas int32) (prevRelicas int32, err error) {
	prevRelicas, err = k.updateReplicasAppsV1(expReplicas)
	if err == nil || !apierrors.IsForbidden(err) {
		return prevRelicas, err
	}
	glog.V(1).Infof("Falling back to extensions/v1beta1, error using apps/v1: %v", err)

	// Fall back to using the extensions API if we get a forbidden error
	scale, err := k.getScaleExtensionsV1beta1(k.target)
	if err != nil {
		return 0, err
	}
	prevRelicas = scale.Spec.Replicas
	if expReplicas != prevRelicas {
		glog.V(0).Infof("Cluster status: SchedulableNodes[%v], SchedulableCores[%v]", k.clusterStatus.SchedulableNodes, k.clusterStatus.SchedulableCores)
		glog.V(0).Infof("Replicas are not as expected : updating replicas from %d to %d", prevRelicas, expReplicas)
		scale.Spec.Replicas = expReplicas
		_, err = k.updateScaleExtensionsV1beta1(k.target, scale)
		if err != nil {
			return 0, err
		}
	}
	return prevRelicas, nil
}

func (k *k8sClient) getScaleExtensionsV1beta1(target *scaleTarget) (*extensionsv1beta1.Scale, error) {
	opt := metav1.GetOptions{}
	switch strings.ToLower(target.kind) {
	case "deployment", "deployments":
		return k.clientset.ExtensionsV1beta1().Deployments(target.namespace).GetScale(target.name, opt)
	case "replicaset", "replicasets":
		return k.clientset.ExtensionsV1beta1().ReplicaSets(target.namespace).GetScale(target.name, opt)
	default:
		return nil, fmt.Errorf("unsupported target kind: %v", target.kind)
	}
}

func (k *k8sClient) updateScaleExtensionsV1beta1(target *scaleTarget, scale *extensionsv1beta1.Scale) (*extensionsv1beta1.Scale, error) {
	switch strings.ToLower(target.kind) {
	case "deployment", "deployments":
		return k.clientset.ExtensionsV1beta1().Deployments(target.namespace).UpdateScale(target.name, scale)
	case "replicaset", "replicasets":
		return k.clientset.ExtensionsV1beta1().ReplicaSets(target.namespace).UpdateScale(target.name, scale)
	default:
		return nil, fmt.Errorf("unsupported target kind: %v", target.kind)
	}
}

func (k *k8sClient) updateReplicasAppsV1(expReplicas int32) (prevRelicas int32, err error) {
	req, err := requestForTarget(k.clientset.AppsV1().RESTClient().Get(), k.target)
	if err != nil {
		return 0, err
	}

	scale := &autoscalingv1.Scale{}
	if err = req.Do().Into(scale); err != nil {
		return 0, err
	}

	prevRelicas = scale.Spec.Replicas
	if expReplicas != prevRelicas {
		glog.V(0).Infof("Cluster status: SchedulableNodes[%v], SchedulableCores[%v]", k.clusterStatus.SchedulableNodes, k.clusterStatus.SchedulableCores)
		glog.V(0).Infof("Replicas are not as expected : updating replicas from %d to %d", prevRelicas, expReplicas)
		scale.Spec.Replicas = expReplicas
		req, err = requestForTarget(k.clientset.AppsV1().RESTClient().Put(), k.target)
		if err != nil {
			return 0, err
		}
		if err = req.Body(scale).Do().Error(); err != nil {
			return 0, err
		}
	}

	return prevRelicas, nil
}

func requestForTarget(req *rest.Request, target *scaleTarget) (*rest.Request, error) {
	var absPath, resource string
	// Support the kinds we allowed scaling via the extensions API group
	// TODO: switch to use the polymorphic scale client once client-go versions are updated
	switch strings.ToLower(target.kind) {
	case "deployment", "deployments":
		absPath = "/apis/apps/v1"
		resource = "deployments"
	case "replicaset", "replicasets":
		absPath = "/apis/apps/v1"
		resource = "replicasets"
	case "statefulset", "statefulsets":
		absPath = "/apis/apps/v1"
		resource = "statefulsets"
	case "replicationcontroller", "replicationcontrollers":
		absPath = "/api/v1"
		resource = "replicationcontrollers"
	default:
		return nil, fmt.Errorf("unsupported target kind: %v", target.kind)
	}

	return req.AbsPath(absPath).Namespace(target.namespace).Resource(resource).Name(target.name).SubResource("scale"), nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

package siegreader

import "os"

// no mmap on appengine
func mmapable(sz int64) bool {
	return false
}

func mmapFile(f *os.File) []byte {
	return nil
}
<commit_msg>app engine fix<commit_after>// Copyright 2014 Richard Lehane. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

package siegreader

import "os"

// no mmap on appengine
func mmapable(sz int64) bool {
	return false
}

func (m *mmap) mapFile() error {
	return nil
}

func (m *mmap) unmap() error {
	return nil
}
<|endoftext|>


<commit_before>package handler

import (
	"encoding/json"
	"net/http"

	"github.com/sirupsen/logrus"
	"github.com/skygeario/skygear-server/pkg/core/auth/authz"
	"github.com/skygeario/skygear-server/pkg/core/auth/authz/policy"
	"github.com/skygeario/skygear-server/pkg/core/db"
	"github.com/skygeario/skygear-server/pkg/core/handler"
	"github.com/skygeario/skygear-server/pkg/core/inject"
	"github.com/skygeario/skygear-server/pkg/core/server"
	recordGear "github.com/skygeario/skygear-server/pkg/record"
	"github.com/skygeario/skygear-server/pkg/record/dependency/record"
)

func AttachCreationAccessHandler(
	server *server.Server,
	recordDependency recordGear.DependencyMap,
) *server.Server {
	server.Handle("/schema/access", &CreationAccessHandlerFactory{
		recordDependency,
	}).Methods("POST")
	return server
}

type CreationAccessHandlerFactory struct {
	Dependency recordGear.DependencyMap
}

func (f CreationAccessHandlerFactory) NewHandler(request *http.Request) http.Handler {
	h := &CreationAccessHandler{}
	inject.DefaultInject(h, f.Dependency, request)
	return handler.APIHandlerToHandler(h, h.TxContext)
}

func (f CreationAccessHandlerFactory) ProvideAuthzPolicy() authz.Policy {
	return policy.AllOf(
		authz.PolicyFunc(policy.RequireMasterKey),
	)
}

type CreationAccessRequestPayload struct {
}

func (s CreationAccessRequestPayload) Validate() error {
	return nil
}

/*
CreationAccessHandler handles the update of creation access of record
curl -X POST -H "Content-Type: application/json" \
  -d @- http://localhost:3000/schema/access <<EOF
{
	"type": "note",
	"create_roles": [
		"admin",
		"writer"
	]
}
EOF
*/
type CreationAccessHandler struct {
	TxContext   db.TxContext  `dependency:"TxContext"`
	RecordStore record.Store  `dependency:"RecordStore"`
	Logger      *logrus.Entry `dependency:"HandlerLogger"`
}

func (h CreationAccessHandler) WithTx() bool {
	return true
}

func (h CreationAccessHandler) DecodeRequest(request *http.Request) (handler.RequestPayload, error) {
	payload := CreationAccessRequestPayload{}
	if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
		return nil, err
	}

	return payload, nil
}

func (h CreationAccessHandler) Handle(req interface{}) (resp interface{}, err error) {
	return
}
<commit_msg>Decode and validate creation access req payload<commit_after>package handler

import (
	"encoding/json"
	"net/http"

	"github.com/sirupsen/logrus"
	"github.com/skygeario/skygear-server/pkg/core/auth/authz"
	"github.com/skygeario/skygear-server/pkg/core/auth/authz/policy"
	"github.com/skygeario/skygear-server/pkg/core/db"
	"github.com/skygeario/skygear-server/pkg/core/handler"
	"github.com/skygeario/skygear-server/pkg/core/inject"
	"github.com/skygeario/skygear-server/pkg/core/server"
	recordGear "github.com/skygeario/skygear-server/pkg/record"
	"github.com/skygeario/skygear-server/pkg/record/dependency/record"
	"github.com/skygeario/skygear-server/pkg/server/skyerr"
)

func AttachCreationAccessHandler(
	server *server.Server,
	recordDependency recordGear.DependencyMap,
) *server.Server {
	server.Handle("/schema/access", &CreationAccessHandlerFactory{
		recordDependency,
	}).Methods("POST")
	return server
}

type CreationAccessHandlerFactory struct {
	Dependency recordGear.DependencyMap
}

func (f CreationAccessHandlerFactory) NewHandler(request *http.Request) http.Handler {
	h := &CreationAccessHandler{}
	inject.DefaultInject(h, f.Dependency, request)
	return handler.APIHandlerToHandler(h, h.TxContext)
}

func (f CreationAccessHandlerFactory) ProvideAuthzPolicy() authz.Policy {
	return policy.AllOf(
		authz.PolicyFunc(policy.RequireMasterKey),
	)
}

type CreationAccessRequestPayload struct {
	Type           string   `json:"type"`
	RawCreateRoles []string `json:"create_roles"`
	ACL            record.ACL
}

func (p CreationAccessRequestPayload) Validate() error {
	if p.Type == "" {
		return skyerr.NewInvalidArgument("missing required fields", []string{"type"})
	}

	return nil
}

/*
CreationAccessHandler handles the update of creation access of record
curl -X POST -H "Content-Type: application/json" \
  -d @- http://localhost:3000/schema/access <<EOF
{
	"type": "note",
	"create_roles": [
		"admin",
		"writer"
	]
}
EOF
*/
type CreationAccessHandler struct {
	TxContext   db.TxContext  `dependency:"TxContext"`
	RecordStore record.Store  `dependency:"RecordStore"`
	Logger      *logrus.Entry `dependency:"HandlerLogger"`
}

func (h CreationAccessHandler) WithTx() bool {
	return true
}

func (h CreationAccessHandler) DecodeRequest(request *http.Request) (handler.RequestPayload, error) {
	payload := CreationAccessRequestPayload{}
	if err := json.NewDecoder(request.Body).Decode(&payload); err != nil {
		return nil, err
	}

	acl := record.ACL{}
	for _, perRoleName := range payload.RawCreateRoles {
		acl = append(acl, record.NewACLEntryRole(perRoleName, record.CreateLevel))
	}

	payload.ACL = acl

	return payload, nil
}

func (h CreationAccessHandler) Handle(req interface{}) (resp interface{}, err error) {
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package service

import (
	"context"
	"fmt"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"k8s.io/apiserver/pkg/storage/names"
	"k8s.io/kubernetes/pkg/api/legacyscheme"
	api "k8s.io/kubernetes/pkg/apis/core"
	"k8s.io/kubernetes/pkg/apis/core/validation"
)

// svcStrategy implements behavior for Services
type svcStrategy struct {
	runtime.ObjectTyper
	names.NameGenerator
}

// Services is the default logic that applies when creating and updating Service
// objects.
var Strategy = svcStrategy{legacyscheme.Scheme, names.SimpleNameGenerator}

// NamespaceScoped is true for services.
func (svcStrategy) NamespaceScoped() bool {
	return true
}

// PrepareForCreate clears fields that are not allowed to be set by end users on creation.
func (svcStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) {
	service := obj.(*api.Service)
	service.Status = api.ServiceStatus{}
}

// PrepareForUpdate clears fields that are not allowed to be set by end users on update.
func (svcStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
	newService := obj.(*api.Service)
	oldService := old.(*api.Service)
	newService.Status = oldService.Status
}

// Validate validates a new service.
func (svcStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList {
	service := obj.(*api.Service)
	allErrs := validation.ValidateService(service)
	allErrs = append(allErrs, validation.ValidateConditionalService(service, nil)...)
	return allErrs
}

// Canonicalize normalizes the object after validation.
func (svcStrategy) Canonicalize(obj runtime.Object) {
}

func (svcStrategy) AllowCreateOnUpdate() bool {
	return true
}

func (svcStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
	allErrs := validation.ValidateServiceUpdate(obj.(*api.Service), old.(*api.Service))
	allErrs = append(allErrs, validation.ValidateConditionalService(obj.(*api.Service), old.(*api.Service))...)
	return allErrs
}

func (svcStrategy) AllowUnconditionalUpdate() bool {
	return true
}

func (svcStrategy) Export(ctx context.Context, obj runtime.Object, exact bool) error {
	t, ok := obj.(*api.Service)
	if !ok {
		// unexpected programmer error
		return fmt.Errorf("unexpected object: %v", obj)
	}
	// TODO: service does not yet have a prepare create strategy (see above)
	t.Status = api.ServiceStatus{}
	if exact {
		return nil
	}
	if t.Spec.ClusterIP != api.ClusterIPNone {
		t.Spec.ClusterIP = ""
	}
	if t.Spec.Type == api.ServiceTypeNodePort {
		for i := range t.Spec.Ports {
			t.Spec.Ports[i].NodePort = 0
		}
	}
	return nil
}

type serviceStatusStrategy struct {
	svcStrategy
}

// StatusStrategy is the default logic invoked when updating service status.
var StatusStrategy = serviceStatusStrategy{Strategy}

// PrepareForUpdate clears fields that are not allowed to be set by end users on update of status
func (serviceStatusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
	newService := obj.(*api.Service)
	oldService := old.(*api.Service)
	// status changes are not allowed to update spec
	newService.Spec = oldService.Spec
}

// ValidateUpdate is the default update validation for an end user updating status
func (serviceStatusStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
	return validation.ValidateServiceStatusUpdate(obj.(*api.Service), old.(*api.Service))
}
<commit_msg>Add dropDisbledFields() to service<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package service

import (
	"context"
	"fmt"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"k8s.io/apiserver/pkg/storage/names"
	"k8s.io/kubernetes/pkg/api/legacyscheme"
	api "k8s.io/kubernetes/pkg/apis/core"
	"k8s.io/kubernetes/pkg/apis/core/validation"
)

// svcStrategy implements behavior for Services
type svcStrategy struct {
	runtime.ObjectTyper
	names.NameGenerator
}

// Services is the default logic that applies when creating and updating Service
// objects.
var Strategy = svcStrategy{legacyscheme.Scheme, names.SimpleNameGenerator}

// NamespaceScoped is true for services.
func (svcStrategy) NamespaceScoped() bool {
	return true
}

// PrepareForCreate clears fields that are not allowed to be set by end users on creation.
func (svcStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) {
	service := obj.(*api.Service)
	service.Status = api.ServiceStatus{}

	dropServiceDisabledFields(service, nil)
}

// PrepareForUpdate clears fields that are not allowed to be set by end users on update.
func (svcStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
	newService := obj.(*api.Service)
	oldService := old.(*api.Service)
	newService.Status = oldService.Status

	dropServiceDisabledFields(newService, oldService)
}

// Validate validates a new service.
func (svcStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList {
	service := obj.(*api.Service)
	allErrs := validation.ValidateService(service)
	allErrs = append(allErrs, validation.ValidateConditionalService(service, nil)...)
	return allErrs
}

// Canonicalize normalizes the object after validation.
func (svcStrategy) Canonicalize(obj runtime.Object) {
}

func (svcStrategy) AllowCreateOnUpdate() bool {
	return true
}

func (svcStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
	allErrs := validation.ValidateServiceUpdate(obj.(*api.Service), old.(*api.Service))
	allErrs = append(allErrs, validation.ValidateConditionalService(obj.(*api.Service), old.(*api.Service))...)
	return allErrs
}

func (svcStrategy) AllowUnconditionalUpdate() bool {
	return true
}

func (svcStrategy) Export(ctx context.Context, obj runtime.Object, exact bool) error {
	t, ok := obj.(*api.Service)
	if !ok {
		// unexpected programmer error
		return fmt.Errorf("unexpected object: %v", obj)
	}
	// TODO: service does not yet have a prepare create strategy (see above)
	t.Status = api.ServiceStatus{}
	if exact {
		return nil
	}
	if t.Spec.ClusterIP != api.ClusterIPNone {
		t.Spec.ClusterIP = ""
	}
	if t.Spec.Type == api.ServiceTypeNodePort {
		for i := range t.Spec.Ports {
			t.Spec.Ports[i].NodePort = 0
		}
	}
	return nil
}

// dropServiceDisabledFields drops fields that are not used if their associated feature gates
// are not enabled.  The typical pattern is:
//     if !utilfeature.DefaultFeatureGate.Enabled(features.MyFeature) && !myFeatureInUse(oldSvc) {
//         newSvc.Spec.MyFeature = nil
//     }
func dropServiceDisabledFields(newSvc *api.Service, oldSvc *api.Service) {
}

type serviceStatusStrategy struct {
	svcStrategy
}

// StatusStrategy is the default logic invoked when updating service status.
var StatusStrategy = serviceStatusStrategy{Strategy}

// PrepareForUpdate clears fields that are not allowed to be set by end users on update of status
func (serviceStatusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
	newService := obj.(*api.Service)
	oldService := old.(*api.Service)
	// status changes are not allowed to update spec
	newService.Spec = oldService.Spec
}

// ValidateUpdate is the default update validation for an end user updating status
func (serviceStatusStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
	return validation.ValidateServiceStatusUpdate(obj.(*api.Service), old.(*api.Service))
}
<|endoftext|>


<commit_before>package sender

import (
	"context"
	"errors"
	"fmt"
	"net/url"
	"sync"
	"time"

	"github.com/benbjohnson/clock"

	"github.com/grafana/grafana/pkg/api/datasource"
	"github.com/grafana/grafana/pkg/infra/log"
	"github.com/grafana/grafana/pkg/services/datasources"
	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
	"github.com/grafana/grafana/pkg/services/ngalert/models"
	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
	"github.com/grafana/grafana/pkg/services/ngalert/store"
	"github.com/grafana/grafana/pkg/services/secrets"
)

// AlertsRouter handles alerts generated during alert rule evaluation.
// Based on rule's orgID and the configuration for that organization,
// it determines whether an alert needs to be sent to an external Alertmanager and\or internal notifier.Alertmanager
//
// After creating a AlertsRouter, you must call Run to keep the AlertsRouter's
// state synchronized with the alerting configuration.
type AlertsRouter struct {
	logger           log.Logger
	clock            clock.Clock
	adminConfigStore store.AdminConfigurationStore

	// externalAlertmanagers help us send alerts to external Alertmanagers.
	adminConfigMtx               sync.RWMutex
	sendAlertsTo                 map[int64]models.AlertmanagersChoice
	externalAlertmanagers        map[int64]*ExternalAlertmanager
	externalAlertmanagersCfgHash map[int64]string

	multiOrgNotifier *notifier.MultiOrgAlertmanager

	appURL                  *url.URL
	disabledOrgs            map[int64]struct{}
	adminConfigPollInterval time.Duration

	datasourceService datasources.DataSourceService
	secretService     secrets.Service
}

func NewAlertsRouter(multiOrgNotifier *notifier.MultiOrgAlertmanager, store store.AdminConfigurationStore,
	clk clock.Clock, appURL *url.URL, disabledOrgs map[int64]struct{}, configPollInterval time.Duration,
	datasourceService datasources.DataSourceService, secretService secrets.Service) *AlertsRouter {
	d := &AlertsRouter{
		logger:           log.New("alerts-router"),
		clock:            clk,
		adminConfigStore: store,

		adminConfigMtx:               sync.RWMutex{},
		externalAlertmanagers:        map[int64]*ExternalAlertmanager{},
		externalAlertmanagersCfgHash: map[int64]string{},
		sendAlertsTo:                 map[int64]models.AlertmanagersChoice{},

		multiOrgNotifier: multiOrgNotifier,

		appURL:                  appURL,
		disabledOrgs:            disabledOrgs,
		adminConfigPollInterval: configPollInterval,

		datasourceService: datasourceService,
		secretService:     secretService,
	}
	return d
}

// SyncAndApplyConfigFromDatabase looks for the admin configuration in the database
// and adjusts the sender(s) and alert handling mechanism accordingly.
func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
	d.logger.Debug("start of admin configuration sync")
	cfgs, err := d.adminConfigStore.GetAdminConfigurations()
	if err != nil {
		return err
	}

	d.logger.Debug("found admin configurations", "count", len(cfgs))

	orgsFound := make(map[int64]struct{}, len(cfgs))
	d.adminConfigMtx.Lock()
	for _, cfg := range cfgs {
		_, isDisabledOrg := d.disabledOrgs[cfg.OrgID]
		if isDisabledOrg {
			d.logger.Debug("skipping starting sender for disabled org", "org", cfg.OrgID)
			continue
		}

		// Update the Alertmanagers choice for the organization.
		d.sendAlertsTo[cfg.OrgID] = cfg.SendAlertsTo

		orgsFound[cfg.OrgID] = struct{}{} // keep track of the which externalAlertmanagers we need to keep.

		existing, ok := d.externalAlertmanagers[cfg.OrgID]

		//  We have no running sender and alerts are handled internally, no-op.
		if !ok && cfg.SendAlertsTo == models.InternalAlertmanager {
			d.logger.Debug("alerts are handled internally", "org", cfg.OrgID)
			continue
		}

		externalAlertmanagers, err := d.alertmanagersFromDatasources(cfg.OrgID)
		if err != nil {
			d.logger.Error("failed to get alertmanagers from datasources",
				"org", cfg.OrgID,
				"err", err)
			continue
		}
		cfg.Alertmanagers = append(cfg.Alertmanagers, externalAlertmanagers...)

		// We have no running sender and no Alertmanager(s) configured, no-op.
		if !ok && len(cfg.Alertmanagers) == 0 {
			d.logger.Debug("no external alertmanagers configured", "org", cfg.OrgID)
			continue
		}

		// We have a running sender but no Alertmanager(s) configured, shut it down.
		if ok && len(cfg.Alertmanagers) == 0 {
			d.logger.Debug("no external alertmanager(s) configured, sender will be stopped", "org", cfg.OrgID)
			delete(orgsFound, cfg.OrgID)
			continue
		}

		d.logger.Debug("alertmanagers found in the configuration", "alertmanagers", cfg.Alertmanagers)

		// We have a running sender, check if we need to apply a new config.
		if ok {
			if d.externalAlertmanagersCfgHash[cfg.OrgID] == cfg.AsSHA256() {
				d.logger.Debug("sender configuration is the same as the one running, no-op", "org", cfg.OrgID, "alertmanagers", cfg.Alertmanagers)
				continue
			}

			d.logger.Debug("applying new configuration to sender", "org", cfg.OrgID, "alertmanagers", cfg.Alertmanagers)
			err := existing.ApplyConfig(cfg)
			if err != nil {
				d.logger.Error("failed to apply configuration", "err", err, "org", cfg.OrgID)
				continue
			}
			d.externalAlertmanagersCfgHash[cfg.OrgID] = cfg.AsSHA256()
			continue
		}

		// No sender and have Alertmanager(s) to send to - start a new one.
		d.logger.Info("creating new sender for the external alertmanagers", "org", cfg.OrgID, "alertmanagers", cfg.Alertmanagers)
		s, err := NewExternalAlertmanagerSender()
		if err != nil {
			d.logger.Error("unable to start the sender", "err", err, "org", cfg.OrgID)
			continue
		}

		d.externalAlertmanagers[cfg.OrgID] = s
		s.Run()

		err = s.ApplyConfig(cfg)
		if err != nil {
			d.logger.Error("failed to apply configuration", "err", err, "org", cfg.OrgID)
			continue
		}

		d.externalAlertmanagersCfgHash[cfg.OrgID] = cfg.AsSHA256()
	}

	sendersToStop := map[int64]*ExternalAlertmanager{}

	for orgID, s := range d.externalAlertmanagers {
		if _, exists := orgsFound[orgID]; !exists {
			sendersToStop[orgID] = s
			delete(d.externalAlertmanagers, orgID)
			delete(d.externalAlertmanagersCfgHash, orgID)
		}
	}
	d.adminConfigMtx.Unlock()

	// We can now stop these externalAlertmanagers w/o having to hold a lock.
	for orgID, s := range sendersToStop {
		d.logger.Info("stopping sender", "org", orgID)
		s.Stop()
		d.logger.Info("stopped sender", "org", orgID)
	}

	d.logger.Debug("finish of admin configuration sync")

	return nil
}

func (d *AlertsRouter) alertmanagersFromDatasources(orgID int64) ([]string, error) {
	var alertmanagers []string
	// We might have alertmanager datasources that are acting as external
	// alertmanager, let's fetch them.
	query := &datasources.GetDataSourcesByTypeQuery{
		OrgId: orgID,
		Type:  datasources.DS_ALERTMANAGER,
	}
	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
	defer cancel()
	err := d.datasourceService.GetDataSourcesByType(ctx, query)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch datasources for org: %w", err)
	}
	for _, ds := range query.Result {
		if !ds.JsonData.Get(definitions.HandleGrafanaManagedAlerts).MustBool(false) {
			continue
		}
		amURL, err := d.buildExternalURL(ds)
		if err != nil {
			d.logger.Error("failed to build external alertmanager URL",
				"org", ds.OrgId,
				"uid", ds.Uid,
				"err", err)
			continue
		}
		alertmanagers = append(alertmanagers, amURL)
	}
	return alertmanagers, nil
}

func (d *AlertsRouter) buildExternalURL(ds *datasources.DataSource) (string, error) {
	// We re-use the same parsing logic as the datasource to make sure it matches whatever output the user received
	// when doing the healthcheck.
	parsed, err := datasource.ValidateURL(datasources.DS_ALERTMANAGER, ds.Url)
	if err != nil {
		return "", fmt.Errorf("failed to parse alertmanager datasource url: %w", err)
	}
	// if basic auth is enabled we need to build the url with basic auth baked in
	if !ds.BasicAuth {
		return parsed.String(), nil
	}

	password := d.secretService.GetDecryptedValue(context.Background(), ds.SecureJsonData, "basicAuthPassword", "")
	if password == "" {
		return "", fmt.Errorf("basic auth enabled but no password set")
	}
	return fmt.Sprintf("%s://%s:%s@%s%s%s", parsed.Scheme, ds.BasicAuthUser,
		password, parsed.Host, parsed.Path, parsed.RawQuery), nil
}

func (d *AlertsRouter) Send(key models.AlertRuleKey, alerts definitions.PostableAlerts) {
	logger := d.logger.New("rule_uid", key.UID, "org", key.OrgID)
	if len(alerts.PostableAlerts) == 0 {
		logger.Debug("no alerts to notify about")
		return
	}
	// Send alerts to local notifier if they need to be handled internally
	// or if no external AMs have been discovered yet.
	var localNotifierExist, externalNotifierExist bool
	if d.sendAlertsTo[key.OrgID] == models.ExternalAlertmanagers && len(d.AlertmanagersFor(key.OrgID)) > 0 {
		logger.Debug("no alerts to put in the notifier")
	} else {
		logger.Debug("sending alerts to local notifier", "count", len(alerts.PostableAlerts), "alerts", alerts.PostableAlerts)
		n, err := d.multiOrgNotifier.AlertmanagerFor(key.OrgID)
		if err == nil {
			localNotifierExist = true
			if err := n.PutAlerts(alerts); err != nil {
				logger.Error("failed to put alerts in the local notifier", "count", len(alerts.PostableAlerts), "err", err)
			}
		} else {
			if errors.Is(err, notifier.ErrNoAlertmanagerForOrg) {
				logger.Debug("local notifier was not found")
			} else {
				logger.Error("local notifier is not available", "err", err)
			}
		}
	}

	// Send alerts to external Alertmanager(s) if we have a sender for this organization
	// and alerts are not being handled just internally.
	d.adminConfigMtx.RLock()
	defer d.adminConfigMtx.RUnlock()
	s, ok := d.externalAlertmanagers[key.OrgID]
	if ok && d.sendAlertsTo[key.OrgID] != models.InternalAlertmanager {
		logger.Debug("sending alerts to external notifier", "count", len(alerts.PostableAlerts), "alerts", alerts.PostableAlerts)
		s.SendAlerts(alerts)
		externalNotifierExist = true
	}

	if !localNotifierExist && !externalNotifierExist {
		logger.Error("no external or internal notifier - [%d] alerts not delivered", len(alerts.PostableAlerts))
	}
}

// AlertmanagersFor returns all the discovered Alertmanager(s) for a particular organization.
func (d *AlertsRouter) AlertmanagersFor(orgID int64) []*url.URL {
	d.adminConfigMtx.RLock()
	defer d.adminConfigMtx.RUnlock()
	s, ok := d.externalAlertmanagers[orgID]
	if !ok {
		return []*url.URL{}
	}
	return s.Alertmanagers()
}

// DroppedAlertmanagersFor returns all the dropped Alertmanager(s) for a particular organization.
func (d *AlertsRouter) DroppedAlertmanagersFor(orgID int64) []*url.URL {
	d.adminConfigMtx.RLock()
	defer d.adminConfigMtx.RUnlock()
	s, ok := d.externalAlertmanagers[orgID]
	if !ok {
		return []*url.URL{}
	}

	return s.DroppedAlertmanagers()
}

// Run starts regular updates of the configuration.
func (d *AlertsRouter) Run(ctx context.Context) error {
	for {
		select {
		case <-time.After(d.adminConfigPollInterval):
			if err := d.SyncAndApplyConfigFromDatabase(); err != nil {
				d.logger.Error("unable to sync admin configuration", "err", err)
			}
		case <-ctx.Done():
			// Stop sending alerts to all external Alertmanager(s).
			d.adminConfigMtx.Lock()
			for orgID, s := range d.externalAlertmanagers {
				delete(d.externalAlertmanagers, orgID) // delete before we stop to make sure we don't accept any more alerts.
				s.Stop()
			}
			d.adminConfigMtx.Unlock()

			return nil
		}
	}
}
<commit_msg>Alerting: log external alertmanager URLs #54127<commit_after>package sender

import (
	"context"
	"errors"
	"fmt"
	"net/url"
	"sync"
	"time"

	"github.com/benbjohnson/clock"

	"github.com/grafana/grafana/pkg/api/datasource"
	"github.com/grafana/grafana/pkg/infra/log"
	"github.com/grafana/grafana/pkg/services/datasources"
	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
	"github.com/grafana/grafana/pkg/services/ngalert/models"
	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
	"github.com/grafana/grafana/pkg/services/ngalert/store"
	"github.com/grafana/grafana/pkg/services/secrets"
)

// AlertsRouter handles alerts generated during alert rule evaluation.
// Based on rule's orgID and the configuration for that organization,
// it determines whether an alert needs to be sent to an external Alertmanager and\or internal notifier.Alertmanager
//
// After creating a AlertsRouter, you must call Run to keep the AlertsRouter's
// state synchronized with the alerting configuration.
type AlertsRouter struct {
	logger           log.Logger
	clock            clock.Clock
	adminConfigStore store.AdminConfigurationStore

	// externalAlertmanagers help us send alerts to external Alertmanagers.
	adminConfigMtx               sync.RWMutex
	sendAlertsTo                 map[int64]models.AlertmanagersChoice
	externalAlertmanagers        map[int64]*ExternalAlertmanager
	externalAlertmanagersCfgHash map[int64]string

	multiOrgNotifier *notifier.MultiOrgAlertmanager

	appURL                  *url.URL
	disabledOrgs            map[int64]struct{}
	adminConfigPollInterval time.Duration

	datasourceService datasources.DataSourceService
	secretService     secrets.Service
}

func NewAlertsRouter(multiOrgNotifier *notifier.MultiOrgAlertmanager, store store.AdminConfigurationStore,
	clk clock.Clock, appURL *url.URL, disabledOrgs map[int64]struct{}, configPollInterval time.Duration,
	datasourceService datasources.DataSourceService, secretService secrets.Service) *AlertsRouter {
	d := &AlertsRouter{
		logger:           log.New("alerts-router"),
		clock:            clk,
		adminConfigStore: store,

		adminConfigMtx:               sync.RWMutex{},
		externalAlertmanagers:        map[int64]*ExternalAlertmanager{},
		externalAlertmanagersCfgHash: map[int64]string{},
		sendAlertsTo:                 map[int64]models.AlertmanagersChoice{},

		multiOrgNotifier: multiOrgNotifier,

		appURL:                  appURL,
		disabledOrgs:            disabledOrgs,
		adminConfigPollInterval: configPollInterval,

		datasourceService: datasourceService,
		secretService:     secretService,
	}
	return d
}

// SyncAndApplyConfigFromDatabase looks for the admin configuration in the database
// and adjusts the sender(s) and alert handling mechanism accordingly.
func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
	d.logger.Debug("start of admin configuration sync")
	cfgs, err := d.adminConfigStore.GetAdminConfigurations()
	if err != nil {
		return err
	}

	d.logger.Debug("found admin configurations", "count", len(cfgs))

	orgsFound := make(map[int64]struct{}, len(cfgs))
	d.adminConfigMtx.Lock()
	for _, cfg := range cfgs {
		_, isDisabledOrg := d.disabledOrgs[cfg.OrgID]
		if isDisabledOrg {
			d.logger.Debug("skipping starting sender for disabled org", "org", cfg.OrgID)
			continue
		}

		// Update the Alertmanagers choice for the organization.
		d.sendAlertsTo[cfg.OrgID] = cfg.SendAlertsTo

		orgsFound[cfg.OrgID] = struct{}{} // keep track of the which externalAlertmanagers we need to keep.

		existing, ok := d.externalAlertmanagers[cfg.OrgID]

		//  We have no running sender and alerts are handled internally, no-op.
		if !ok && cfg.SendAlertsTo == models.InternalAlertmanager {
			d.logger.Debug("alerts are handled internally", "org", cfg.OrgID)
			continue
		}

		externalAlertmanagers, err := d.alertmanagersFromDatasources(cfg.OrgID)
		if err != nil {
			d.logger.Error("failed to get alertmanagers from datasources",
				"org", cfg.OrgID,
				"err", err)
			continue
		}
		cfg.Alertmanagers = append(cfg.Alertmanagers, externalAlertmanagers...)

		// We have no running sender and no Alertmanager(s) configured, no-op.
		if !ok && len(cfg.Alertmanagers) == 0 {
			d.logger.Debug("no external alertmanagers configured", "org", cfg.OrgID)
			continue
		}

		// We have a running sender but no Alertmanager(s) configured, shut it down.
		if ok && len(cfg.Alertmanagers) == 0 {
			d.logger.Debug("no external alertmanager(s) configured, sender will be stopped", "org", cfg.OrgID)
			delete(orgsFound, cfg.OrgID)
			continue
		}

		// Avoid logging sensitive data
		var redactedAMs []string
		for _, am := range cfg.Alertmanagers {
			parsedAM, err := url.Parse(am)
			if err != nil {
				d.logger.Error("failed to parse alertmanager string",
					"org", cfg.OrgID,
					"err", err)
				continue
			}
			redactedAMs = append(redactedAMs, parsedAM.Redacted())
		}

		d.logger.Debug("alertmanagers found in the configuration", "alertmanagers", redactedAMs)

		// We have a running sender, check if we need to apply a new config.
		amHash := cfg.AsSHA256()
		if ok {
			if d.externalAlertmanagersCfgHash[cfg.OrgID] == amHash {
				d.logger.Debug("sender configuration is the same as the one running, no-op", "org", cfg.OrgID, "alertmanagers", redactedAMs)
				continue
			}

			d.logger.Debug("applying new configuration to sender", "org", cfg.OrgID, "alertmanagers", redactedAMs)
			err := existing.ApplyConfig(cfg)
			if err != nil {
				d.logger.Error("failed to apply configuration", "err", err, "org", cfg.OrgID)
				continue
			}
			d.externalAlertmanagersCfgHash[cfg.OrgID] = amHash
			continue
		}

		// No sender and have Alertmanager(s) to send to - start a new one.
		d.logger.Info("creating new sender for the external alertmanagers", "org", cfg.OrgID, "alertmanagers", redactedAMs)
		s, err := NewExternalAlertmanagerSender()
		if err != nil {
			d.logger.Error("unable to start the sender", "err", err, "org", cfg.OrgID)
			continue
		}

		d.externalAlertmanagers[cfg.OrgID] = s
		s.Run()

		err = s.ApplyConfig(cfg)
		if err != nil {
			d.logger.Error("failed to apply configuration", "err", err, "org", cfg.OrgID)
			continue
		}

		d.externalAlertmanagersCfgHash[cfg.OrgID] = amHash
	}

	sendersToStop := map[int64]*ExternalAlertmanager{}

	for orgID, s := range d.externalAlertmanagers {
		if _, exists := orgsFound[orgID]; !exists {
			sendersToStop[orgID] = s
			delete(d.externalAlertmanagers, orgID)
			delete(d.externalAlertmanagersCfgHash, orgID)
		}
	}
	d.adminConfigMtx.Unlock()

	// We can now stop these externalAlertmanagers w/o having to hold a lock.
	for orgID, s := range sendersToStop {
		d.logger.Info("stopping sender", "org", orgID)
		s.Stop()
		d.logger.Info("stopped sender", "org", orgID)
	}

	d.logger.Debug("finish of admin configuration sync")

	return nil
}

func (d *AlertsRouter) alertmanagersFromDatasources(orgID int64) ([]string, error) {
	var alertmanagers []string
	// We might have alertmanager datasources that are acting as external
	// alertmanager, let's fetch them.
	query := &datasources.GetDataSourcesByTypeQuery{
		OrgId: orgID,
		Type:  datasources.DS_ALERTMANAGER,
	}
	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
	defer cancel()
	err := d.datasourceService.GetDataSourcesByType(ctx, query)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch datasources for org: %w", err)
	}
	for _, ds := range query.Result {
		if !ds.JsonData.Get(definitions.HandleGrafanaManagedAlerts).MustBool(false) {
			continue
		}
		amURL, err := d.buildExternalURL(ds)
		if err != nil {
			d.logger.Error("failed to build external alertmanager URL",
				"org", ds.OrgId,
				"uid", ds.Uid,
				"err", err)
			continue
		}
		alertmanagers = append(alertmanagers, amURL)
	}
	return alertmanagers, nil
}

func (d *AlertsRouter) buildExternalURL(ds *datasources.DataSource) (string, error) {
	// We re-use the same parsing logic as the datasource to make sure it matches whatever output the user received
	// when doing the healthcheck.
	parsed, err := datasource.ValidateURL(datasources.DS_ALERTMANAGER, ds.Url)
	if err != nil {
		return "", fmt.Errorf("failed to parse alertmanager datasource url: %w", err)
	}
	// if basic auth is enabled we need to build the url with basic auth baked in
	if !ds.BasicAuth {
		return parsed.String(), nil
	}

	password := d.secretService.GetDecryptedValue(context.Background(), ds.SecureJsonData, "basicAuthPassword", "")
	if password == "" {
		return "", fmt.Errorf("basic auth enabled but no password set")
	}
	return fmt.Sprintf("%s://%s:%s@%s%s%s", parsed.Scheme, ds.BasicAuthUser,
		password, parsed.Host, parsed.Path, parsed.RawQuery), nil
}

func (d *AlertsRouter) Send(key models.AlertRuleKey, alerts definitions.PostableAlerts) {
	logger := d.logger.New("rule_uid", key.UID, "org", key.OrgID)
	if len(alerts.PostableAlerts) == 0 {
		logger.Debug("no alerts to notify about")
		return
	}
	// Send alerts to local notifier if they need to be handled internally
	// or if no external AMs have been discovered yet.
	var localNotifierExist, externalNotifierExist bool
	if d.sendAlertsTo[key.OrgID] == models.ExternalAlertmanagers && len(d.AlertmanagersFor(key.OrgID)) > 0 {
		logger.Debug("no alerts to put in the notifier")
	} else {
		logger.Debug("sending alerts to local notifier", "count", len(alerts.PostableAlerts), "alerts", alerts.PostableAlerts)
		n, err := d.multiOrgNotifier.AlertmanagerFor(key.OrgID)
		if err == nil {
			localNotifierExist = true
			if err := n.PutAlerts(alerts); err != nil {
				logger.Error("failed to put alerts in the local notifier", "count", len(alerts.PostableAlerts), "err", err)
			}
		} else {
			if errors.Is(err, notifier.ErrNoAlertmanagerForOrg) {
				logger.Debug("local notifier was not found")
			} else {
				logger.Error("local notifier is not available", "err", err)
			}
		}
	}

	// Send alerts to external Alertmanager(s) if we have a sender for this organization
	// and alerts are not being handled just internally.
	d.adminConfigMtx.RLock()
	defer d.adminConfigMtx.RUnlock()
	s, ok := d.externalAlertmanagers[key.OrgID]
	if ok && d.sendAlertsTo[key.OrgID] != models.InternalAlertmanager {
		logger.Debug("sending alerts to external notifier", "count", len(alerts.PostableAlerts), "alerts", alerts.PostableAlerts)
		s.SendAlerts(alerts)
		externalNotifierExist = true
	}

	if !localNotifierExist && !externalNotifierExist {
		logger.Error("no external or internal notifier - [%d] alerts not delivered", len(alerts.PostableAlerts))
	}
}

// AlertmanagersFor returns all the discovered Alertmanager(s) for a particular organization.
func (d *AlertsRouter) AlertmanagersFor(orgID int64) []*url.URL {
	d.adminConfigMtx.RLock()
	defer d.adminConfigMtx.RUnlock()
	s, ok := d.externalAlertmanagers[orgID]
	if !ok {
		return []*url.URL{}
	}
	return s.Alertmanagers()
}

// DroppedAlertmanagersFor returns all the dropped Alertmanager(s) for a particular organization.
func (d *AlertsRouter) DroppedAlertmanagersFor(orgID int64) []*url.URL {
	d.adminConfigMtx.RLock()
	defer d.adminConfigMtx.RUnlock()
	s, ok := d.externalAlertmanagers[orgID]
	if !ok {
		return []*url.URL{}
	}

	return s.DroppedAlertmanagers()
}

// Run starts regular updates of the configuration.
func (d *AlertsRouter) Run(ctx context.Context) error {
	for {
		select {
		case <-time.After(d.adminConfigPollInterval):
			if err := d.SyncAndApplyConfigFromDatabase(); err != nil {
				d.logger.Error("unable to sync admin configuration", "err", err)
			}
		case <-ctx.Done():
			// Stop sending alerts to all external Alertmanager(s).
			d.adminConfigMtx.Lock()
			for orgID, s := range d.externalAlertmanagers {
				delete(d.externalAlertmanagers, orgID) // delete before we stop to make sure we don't accept any more alerts.
				s.Stop()
			}
			d.adminConfigMtx.Unlock()

			return nil
		}
	}
}
<|endoftext|>


<commit_before>// Package client provides an implementation of a restricted subset of kubernetes API client
package client

import (
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/micro/go-micro/util/log"
)

const (
	// https://github.com/kubernetes/apimachinery/blob/master/pkg/util/validation/validation.go#L134
	dns1123LabelFmt string = "[a-z0-9]([-a-z0-9]*[a-z0-9])?"
)

var (
	// DefaultImage is default micro image
	DefaultImage = "micro/micro"
	// ServiceRegexp is used to validate service name
	ServiceRegexp = regexp.MustCompile("^" + dns1123LabelFmt + "$")
)

// Kubernetes client
type Kubernetes interface {
	// Create creates new API resource
	Create(*Resource) error
	// Get queries API resrouces
	Get(*Resource, map[string]string) error
	// Update patches existing API object
	Update(*Resource) error
	// Delete deletes API resource
	Delete(*Resource) error
	// List lists API resources
	List(*Resource) error
}

// DefaultService returns default micro kubernetes service definition
func DefaultService(name, version string) *Service {
	Labels := map[string]string{
		"name":    name,
		"version": version,
		"micro":   "service",
	}

	svcName := name
	if len(version) > 0 {
		// API service object name joins name and version over "-"
		svcName = strings.Join([]string{name, version}, "-")

	}

	Metadata := &Metadata{
		Name:      svcName,
		Namespace: "default",
		Version:   version,
		Labels:    Labels,
	}

	Spec := &ServiceSpec{
		Type:     "ClusterIP",
		Selector: Labels,
		Ports: []ServicePort{{
			name + "-port", 9090, "",
		}},
	}

	return &Service{
		Metadata: Metadata,
		Spec:     Spec,
	}
}

// DefaultService returns default micro kubernetes deployment definition
func DefaultDeployment(name, version string) *Deployment {
	Labels := map[string]string{
		"name":    name,
		"version": version,
		"micro":   "service",
	}

	// API deployment object name joins name and version over "="
	depName := strings.Join([]string{name, version}, "-")

	Metadata := &Metadata{
		Name:      depName,
		Namespace: "default",
		Version:   version,
		Labels:    Labels,
	}

	// TODO: we need to figure out this version stuff
	// might be worth adding Build to runtime.Service
	buildTime, err := strconv.ParseInt(version, 10, 64)
	if err == nil {
		buildUnixTimeUTC := time.Unix(buildTime, 0)
		Metadata.Annotations = map[string]string{
			"build": buildUnixTimeUTC.Format(time.RFC3339),
		}
	} else {
		log.Debugf("Runtime could not parse build: %v", err)
	}

	// TODO: change the image name here
	Spec := &DeploymentSpec{
		Replicas: 1,
		Selector: &LabelSelector{
			MatchLabels: Labels,
		},
		Template: &Template{
			Metadata: Metadata,
			PodSpec: &PodSpec{
				Containers: []Container{{
					Name:    name,
					Image:   DefaultImage,
					Env:     []EnvVar{},
					Command: []string{"go", "run", "main.go"},
					Ports: []ContainerPort{{
						Name:          name + "-port",
						ContainerPort: 8080,
					}},
				}},
			},
		},
	}

	return &Deployment{
		Metadata: Metadata,
		Spec:     Spec,
	}
}
<commit_msg>Change DefaultImage to micro/go-micro<commit_after>// Package client provides an implementation of a restricted subset of kubernetes API client
package client

import (
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/micro/go-micro/util/log"
)

const (
	// https://github.com/kubernetes/apimachinery/blob/master/pkg/util/validation/validation.go#L134
	dns1123LabelFmt string = "[a-z0-9]([-a-z0-9]*[a-z0-9])?"
)

var (
	// DefaultImage is default micro image
	DefaultImage = "micro/go-micro"
	// ServiceRegexp is used to validate service name
	ServiceRegexp = regexp.MustCompile("^" + dns1123LabelFmt + "$")
)

// Kubernetes client
type Kubernetes interface {
	// Create creates new API resource
	Create(*Resource) error
	// Get queries API resrouces
	Get(*Resource, map[string]string) error
	// Update patches existing API object
	Update(*Resource) error
	// Delete deletes API resource
	Delete(*Resource) error
	// List lists API resources
	List(*Resource) error
}

// DefaultService returns default micro kubernetes service definition
func DefaultService(name, version string) *Service {
	Labels := map[string]string{
		"name":    name,
		"version": version,
		"micro":   "service",
	}

	svcName := name
	if len(version) > 0 {
		// API service object name joins name and version over "-"
		svcName = strings.Join([]string{name, version}, "-")

	}

	Metadata := &Metadata{
		Name:      svcName,
		Namespace: "default",
		Version:   version,
		Labels:    Labels,
	}

	Spec := &ServiceSpec{
		Type:     "ClusterIP",
		Selector: Labels,
		Ports: []ServicePort{{
			name + "-port", 9090, "",
		}},
	}

	return &Service{
		Metadata: Metadata,
		Spec:     Spec,
	}
}

// DefaultService returns default micro kubernetes deployment definition
func DefaultDeployment(name, version string) *Deployment {
	Labels := map[string]string{
		"name":    name,
		"version": version,
		"micro":   "service",
	}

	// API deployment object name joins name and version over "="
	depName := strings.Join([]string{name, version}, "-")

	Metadata := &Metadata{
		Name:      depName,
		Namespace: "default",
		Version:   version,
		Labels:    Labels,
	}

	// TODO: we need to figure out this version stuff
	// might be worth adding Build to runtime.Service
	buildTime, err := strconv.ParseInt(version, 10, 64)
	if err == nil {
		buildUnixTimeUTC := time.Unix(buildTime, 0)
		Metadata.Annotations = map[string]string{
			"build": buildUnixTimeUTC.Format(time.RFC3339),
		}
	} else {
		log.Debugf("Runtime could not parse build: %v", err)
	}

	// TODO: change the image name here
	Spec := &DeploymentSpec{
		Replicas: 1,
		Selector: &LabelSelector{
			MatchLabels: Labels,
		},
		Template: &Template{
			Metadata: Metadata,
			PodSpec: &PodSpec{
				Containers: []Container{{
					Name:    name,
					Image:   DefaultImage,
					Env:     []EnvVar{},
					Command: []string{"go", "run", "main.go"},
					Ports: []ContainerPort{{
						Name:          name + "-port",
						ContainerPort: 8080,
					}},
				}},
			},
		},
	}

	return &Deployment{
		Metadata: Metadata,
		Spec:     Spec,
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Heptio Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logging

import (
	"fmt"
	"runtime"
	"strings"

	"github.com/sirupsen/logrus"
)

const (
	logSourceField          = "logSource"
	logSourceSetMarkerField = "@logSourceSetBy"
)

// LogLocationHook is a logrus hook that attaches location information
// to log entries, i.e. the file and line number of the logrus log call.
// This hook is designed for use in both the Ark server and Ark plugin
// implementations. When triggered within a plugin, a marker field will
// be set on the log entry indicating that the location came from a plugin.
// The Ark server instance will not overwrite location information if
// it sees this marker.
type LogLocationHook struct {
	loggerName string
}

// WithLoggerName gives the hook a name to use when setting the marker field
// on a log entry indicating the location has been recorded by a plugin. This
// should only be used when setting up a hook for a logger used in a plugin.
func (h *LogLocationHook) WithLoggerName(name string) *LogLocationHook {
	h.loggerName = name
	return h
}

func (h *LogLocationHook) Levels() []logrus.Level {
	return logrus.AllLevels
}

func (h *LogLocationHook) Fire(entry *logrus.Entry) error {
	pcs := make([]uintptr, 64)

	// skip 2 frames:
	//   runtime.Callers
	//   github.com/heptio/ark/pkg/util/logging/(*LogLocationHook).Fire
	n := runtime.Callers(2, pcs)

	// re-slice pcs based on the number of entries written
	frames := runtime.CallersFrames(pcs[:n])

	// now traverse up the call stack looking for the first non-logrus
	// func, which will be the logrus invoker
	var (
		frame runtime.Frame
		more  = true
	)

	for more {
		frame, more = frames.Next()

		if strings.Contains(frame.File, "github.com/sirupsen/logrus") {
			continue
		}

		// set the marker field if we're within a plugin indicating that
		// the location comes from the plugin.
		if h.loggerName != "" {
			entry.Data[logSourceSetMarkerField] = h.loggerName
		}

		// record the log statement location if we're within a plugin OR if
		// we're in Ark server and not logging something that has the marker
		// set (which would indicate the log statement is coming from a plugin).
		if h.loggerName != "" || getLogSourceSetMarker(entry) == "" {
			entry.Data[logSourceField] = fmt.Sprintf("%s:%d", frame.File, frame.Line)
		}

		// if we're in the Ark server, remove the marker field since we don't
		// want to record it in the actual log.
		if h.loggerName == "" {
			delete(entry.Data, logSourceSetMarkerField)
		}

		break
	}

	return nil
}

func getLogSourceSetMarker(entry *logrus.Entry) string {
	nameVal, found := entry.Data[logSourceSetMarkerField]
	if !found {
		return ""
	}

	if name, ok := nameVal.(string); ok {
		return name
	}

	return fmt.Sprintf("%s", nameVal)
}
<commit_msg>Make logSource more concise<commit_after>/*
Copyright 2017 Heptio Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logging

import (
	"fmt"
	"runtime"
	"strings"

	"github.com/sirupsen/logrus"
)

const (
	logSourceField          = "logSource"
	logSourceSetMarkerField = "@logSourceSetBy"
	logrusPackage           = "github.com/sirupsen/logrus"
	arkPackage              = "github.com/heptio/ark"
	arkPackageLen           = len(arkPackage)
)

// LogLocationHook is a logrus hook that attaches location information
// to log entries, i.e. the file and line number of the logrus log call.
// This hook is designed for use in both the Ark server and Ark plugin
// implementations. When triggered within a plugin, a marker field will
// be set on the log entry indicating that the location came from a plugin.
// The Ark server instance will not overwrite location information if
// it sees this marker.
type LogLocationHook struct {
	loggerName string
}

// WithLoggerName gives the hook a name to use when setting the marker field
// on a log entry indicating the location has been recorded by a plugin. This
// should only be used when setting up a hook for a logger used in a plugin.
func (h *LogLocationHook) WithLoggerName(name string) *LogLocationHook {
	h.loggerName = name
	return h
}

func (h *LogLocationHook) Levels() []logrus.Level {
	return logrus.AllLevels
}

func (h *LogLocationHook) Fire(entry *logrus.Entry) error {
	pcs := make([]uintptr, 64)

	// skip 2 frames:
	//   runtime.Callers
	//   github.com/heptio/ark/pkg/util/logging/(*LogLocationHook).Fire
	n := runtime.Callers(2, pcs)

	// re-slice pcs based on the number of entries written
	frames := runtime.CallersFrames(pcs[:n])

	// now traverse up the call stack looking for the first non-logrus
	// func, which will be the logrus invoker
	var (
		frame runtime.Frame
		more  = true
	)

	for more {
		frame, more = frames.Next()

		if strings.Contains(frame.File, logrusPackage) {
			continue
		}

		// set the marker field if we're within a plugin indicating that
		// the location comes from the plugin.
		if h.loggerName != "" {
			entry.Data[logSourceSetMarkerField] = h.loggerName
		}

		// record the log statement location if we're within a plugin OR if
		// we're in Ark server and not logging something that has the marker
		// set (which would indicate the log statement is coming from a plugin).
		if h.loggerName != "" || getLogSourceSetMarker(entry) == "" {
			file := frame.File
			if index := strings.Index(file, arkPackage); index != -1 {
				// strip off .../github.com/heptio/ark/ so we just have pkg/...
				file = frame.File[index+arkPackageLen+1:]
			}

			entry.Data[logSourceField] = fmt.Sprintf("%s:%d", file, frame.Line)
		}

		// if we're in the Ark server, remove the marker field since we don't
		// want to record it in the actual log.
		if h.loggerName == "" {
			delete(entry.Data, logSourceSetMarkerField)
		}

		break
	}

	return nil
}

func getLogSourceSetMarker(entry *logrus.Entry) string {
	nameVal, found := entry.Data[logSourceSetMarkerField]
	if !found {
		return ""
	}

	if name, ok := nameVal.(string); ok {
		return name
	}

	return fmt.Sprintf("%s", nameVal)
}
<|endoftext|>


<commit_before>package dup

import (
	"bytes"
	"fmt"
	"io"
	"strings"
	"text/template"
	"time"

	ledgertools "github.com/ginabythebay/ledger-tools"
)

var compilerTemplate = template.Must(template.New("CompilerOutput").Parse(strings.TrimSpace(`
Possible duplicate {{.One.AmountText}} {{.One.Account}}
	at {{.One.Xact.DateText}} {{.One.Xact.Payee}} ({{.One.Xact.SrcFile}}:{{.One.BegLine}})
	at {{.Two.Xact.DateText}} {{.Two.Xact.Payee}} ({{.Two.Xact.SrcFile}}:{{.Two.BegLine}})
`)))

type key struct {
	account string
	amount  string
	date    string
}

func newKey(account, amount string, t time.Time) key {
	return key{account, amount, t.Format("2006/01/02")}
}

func suppressedDates(notes []string) []string {
	var dates []string
	for _, line := range notes {
		split := strings.SplitAfterN(line, "SuppressDuplicates:", 2)
		if len(split) != 2 {
			continue
		}
		someCandidates := strings.Split(split[1], ",")
		for _, c := range someCandidates {
			c = strings.TrimSpace(c)
			_, err := time.Parse("2006/01/02", c)
			if err == nil {
				dates = append(dates, c)
			}
		}
	}
	return dates
}

func isDateSuppressed(date string, notes []string) bool {
	for _, d := range suppressedDates(notes) {
		if date == d {
			return true
		}
	}
	return false
}

type Pair struct {
	One *ledgertools.Posting
	Two *ledgertools.Posting
}

func (p Pair) CompilerText() (string, error) {
	var buf bytes.Buffer
	err := compilerTemplate.Execute(&buf, p)
	return buf.String(), err
}

func (p Pair) IsSuppressed() bool {
	return isDateSuppressed(p.One.Xact.DateText(), p.Two.Notes) || isDateSuppressed(p.Two.Xact.DateText(), p.One.Notes)
}

// Finder tracks postings and looks for potential duplicates, based on the amount and the date.
type Finder struct {
	// # days to use when looking for matches.  0 means only look for matches on exactly the same day
	Days int

	m map[key][]*ledgertools.Posting

	AllPairs []Pair
}

// NewFinder creates a new Finder.
func NewFinder(days int) *Finder {
	return &Finder{
		Days: days,
		m:    make(map[key][]*ledgertools.Posting),
	}
}

// Add adds p and tracks any existing postings that have the same amount and
// are within the configured number of days.
func (f *Finder) Add(p *ledgertools.Posting) {
	var matches []*ledgertools.Posting
	t := p.Xact.Date
	amount := p.AmountText()
	k := newKey(p.Account, amount, t)

	if f.Days >= 0 {
		matches = append(matches, f.m[k]...)
	}
	for i := 1; i <= f.Days; i++ {
		before := t.AddDate(0, 0, -i)
		matches = append(matches, f.m[newKey(p.Account, amount, before)]...)

		after := t.AddDate(0, 0, i)
		matches = append(matches, f.m[newKey(p.Account, amount, after)]...)
	}

	for _, m := range matches {
		p := Pair{m, p}
		if !p.IsSuppressed() {
			f.AllPairs = append(f.AllPairs, p)
		}
	}

	f.m[k] = append(f.m[k], p)
}

type Writer func() error

func JavacWriter(allPairs []Pair, w io.Writer) Writer {
	return func() error {
		matchCount := 0
		for _, p := range allPairs {
			s, err := p.CompilerText()
			if err != nil {
				return err
			}
			matchCount++

			fmt.Fprintln(w, s)
		}

		fmt.Fprintf(w, "\n %d potential duplicates found\n", matchCount)
		return nil
	}
}
<commit_msg>make SuppressDuplicates parsing more robust<commit_after>package dup

import (
	"bytes"
	"fmt"
	"io"
	"strings"
	"text/template"
	"time"

	ledgertools "github.com/ginabythebay/ledger-tools"
)

var compilerTemplate = template.Must(template.New("CompilerOutput").Parse(strings.TrimSpace(`
Possible duplicate {{.One.AmountText}} {{.One.Account}}
	at {{.One.Xact.DateText}} {{.One.Xact.Payee}} ({{.One.Xact.SrcFile}}:{{.One.BegLine}})
	at {{.Two.Xact.DateText}} {{.Two.Xact.Payee}} ({{.Two.Xact.SrcFile}}:{{.Two.BegLine}})
`)))

type key struct {
	account string
	amount  string
	date    string
}

func newKey(account, amount string, t time.Time) key {
	return key{account, amount, t.Format("2006/01/02")}
}

func suppressedDates(notes []string) []string {
	var dates []string
	for _, line := range notes {
		split := strings.SplitAfterN(line, "SuppressDuplicates:", 2)
		if len(split) != 2 {
			continue
		}
		someCandidates := strings.Split(split[1], ",")
		for _, c := range someCandidates {
			c = strings.TrimSpace(c)
			if len(c) < 10 {
				// dates are 10 characters long.
				break
			}
			_, err := time.Parse("2006/01/02", c[:10])
			if err == nil {
				dates = append(dates, c[:10])
			}
			if len(c) > 10 {
				// sometimes there is glop after a valid date.  If we hit that case, just bail out until the next line
				break
			}
		}
	}

	return dates
}

func isDateSuppressed(date string, notes []string) bool {
	for _, d := range suppressedDates(notes) {
		if date == d {
			return true
		}
	}
	return false
}

type Pair struct {
	One *ledgertools.Posting
	Two *ledgertools.Posting
}

func (p Pair) CompilerText() (string, error) {
	var buf bytes.Buffer
	err := compilerTemplate.Execute(&buf, p)
	return buf.String(), err
}

func (p Pair) IsSuppressed() bool {
	return isDateSuppressed(p.One.Xact.DateText(), p.Two.Notes) || isDateSuppressed(p.Two.Xact.DateText(), p.One.Notes)
}

// Finder tracks postings and looks for potential duplicates, based on the amount and the date.
type Finder struct {
	// # days to use when looking for matches.  0 means only look for matches on exactly the same day
	Days int

	m map[key][]*ledgertools.Posting

	AllPairs []Pair
}

// NewFinder creates a new Finder.
func NewFinder(days int) *Finder {
	return &Finder{
		Days: days,
		m:    make(map[key][]*ledgertools.Posting),
	}
}

// Add adds p and tracks any existing postings that have the same amount and
// are within the configured number of days.
func (f *Finder) Add(p *ledgertools.Posting) {
	var matches []*ledgertools.Posting
	t := p.Xact.Date
	amount := p.AmountText()
	k := newKey(p.Account, amount, t)

	if f.Days >= 0 {
		matches = append(matches, f.m[k]...)
	}
	for i := 1; i <= f.Days; i++ {
		before := t.AddDate(0, 0, -i)
		matches = append(matches, f.m[newKey(p.Account, amount, before)]...)

		after := t.AddDate(0, 0, i)
		matches = append(matches, f.m[newKey(p.Account, amount, after)]...)
	}

	for _, m := range matches {
		p := Pair{m, p}
		if !p.IsSuppressed() {
			f.AllPairs = append(f.AllPairs, p)
		}
	}

	f.m[k] = append(f.m[k], p)
}

type Writer func() error

func JavacWriter(allPairs []Pair, w io.Writer) Writer {
	return func() error {
		matchCount := 0
		for _, p := range allPairs {
			s, err := p.CompilerText()
			if err != nil {
				return err
			}
			matchCount++

			fmt.Fprintln(w, s)
		}

		fmt.Fprintf(w, "\n %d potential duplicates found\n", matchCount)
		return nil
	}
}
<|endoftext|>