<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package modules

import (
	"bytes"
	"testing"

	. "gopkg.in/check.v1"

	"github.com/cilium/cilium/pkg/checker"
)

const (
	modulesContent = `ebtable_nat 16384 1 - Live 0x0000000000000000
ebtable_broute 16384 1 - Live 0x0000000000000000
bridge 172032 1 ebtable_broute, Live 0x0000000000000000
ip6table_nat 16384 1 - Live 0x0000000000000000
nf_nat_ipv6 16384 1 ip6table_nat, Live 0x0000000000000000
ip6table_mangle 16384 1 - Live 0x0000000000000000
ip6table_raw 16384 1 - Live 0x0000000000000000
ip6table_security 16384 1 - Live 0x0000000000000000
iptable_nat 16384 1 - Live 0x0000000000000000
nf_nat_ipv4 16384 1 iptable_nat, Live 0x0000000000000000
iptable_mangle 16384 1 - Live 0x0000000000000000
iptable_raw 16384 1 - Live 0x0000000000000000
iptable_security 16384 1 - Live 0x0000000000000000
ebtable_filter 16384 1 - Live 0x0000000000000000
ebtables 36864 3 ebtable_nat,ebtable_broute,ebtable_filter, Live 0x0000000000000000
ip6table_filter 16384 1 - Live 0x0000000000000000
ip6_tables 28672 5 ip6table_nat,ip6table_mangle,ip6table_raw,ip6table_security,ip6table_filter, Live 0x0000000000000000
iptable_filter 16384 1 - Live 0x0000000000000000
ip_tables 28672 5 iptable_nat,iptable_mangle,iptable_raw,iptable_security,iptable_filter, Live 0x0000000000000000
x_tables 40960 23 xt_multiport,xt_nat,xt_addrtype,xt_mark,xt_comment,xt_CHECKSUM,ipt_MASQUERADE,xt_tcpudp,ip6t_rpfilter,ip6t_REJECT,ipt_REJECT,xt_conntrack,ip6table_mangle,ip6table_raw,ip6table_security,iptable_mangle,iptable_raw,iptable_security,ebtables,ip6table_filter,ip6_tables,iptable_filter,ip_tables, Live 0x0000000000000000`
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) {
	TestingT(t)
}

type ModulesTestSuite struct{}

var _ = Suite(&ModulesTestSuite{})

func (s *ModulesTestSuite) TestInit(c *C) {
	manager := &ModulesManager{}
	c.Assert(manager.modulesList, IsNil)
	err := manager.Init()
	c.Assert(err, IsNil)
	c.Assert(manager.modulesList, NotNil)
}

func (s *ModulesTestSuite) TestFindModules(c *C) {
	manager := &ModulesManager{
		modulesList: []string{
			"ip6_tables",
			"ip6table_mangle",
			"ip6table_filter",
			"ip6table_security",
			"ip6table_raw",
			"ip6table_nat",
		},
	}
	testCases := []struct {
		modulesToFind []string
		isSubset      bool
		expectedDiff  []string
	}{
		{
			modulesToFind: []string{
				"ip6_tables",
				"ip6table_mangle",
				"ip6table_filter",
				"ip6table_security",
				"ip6table_raw",
				"ip6table_nat",
			},
			isSubset:     true,
			expectedDiff: nil,
		},
		{
			modulesToFind: []string{
				"ip6_tables",
				"ip6table_mangle",
				"ip6table_raw",
			},
			isSubset:     true,
			expectedDiff: nil,
		},
		{
			modulesToFind: []string{
				"ip6_tables",
				"ip6table_mangle",
				"ip6table_raw",
				"foo_module",
			},
			isSubset:     false,
			expectedDiff: []string{"foo_module"},
		},
		{
			modulesToFind: []string{
				"foo_module",
				"bar_module",
			},
			isSubset:     false,
			expectedDiff: []string{"foo_module", "bar_module"},
		},
	}
	for _, tc := range testCases {
		found, diff := manager.FindModules(tc.modulesToFind...)
		c.Assert(found, Equals, tc.isSubset)
		c.Assert(diff, checker.DeepEquals, tc.expectedDiff)
	}
}

func (s *ModulesTestSuite) TestParseModuleFile(c *C) {
	expectedLength := 20
	expectedModules := []string{
		"ebtable_nat",
		"ebtable_broute",
		"bridge",
		"ip6table_nat",
		"nf_nat_ipv6",
		"ip6table_mangle",
		"ip6table_raw",
		"ip6table_security",
		"iptable_nat",
		"nf_nat_ipv4",
		"iptable_mangle",
		"iptable_raw",
		"iptable_security",
		"ebtable_filter",
		"ebtables",
		"ip6table_filter",
		"ip6_tables",
		"iptable_filter",
		"ip_tables",
		"x_tables",
	}

	r := bytes.NewBuffer([]byte(modulesContent))
	moduleInfos, err := parseModulesFile(r)
	c.Assert(err, IsNil)
	c.Assert(moduleInfos, HasLen, expectedLength)
	c.Assert(moduleInfos, checker.DeepEquals, expectedModules)
}

func (s *ModulesTestSuite) TestListModules(c *C) {
	_, err := listModules()
	c.Assert(err, IsNil)
}
<commit_msg>modules: pass TestInit on kernels without kernel modules<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package modules

import (
	"bytes"
	"testing"

	. "gopkg.in/check.v1"

	"github.com/cilium/cilium/pkg/checker"
)

const (
	modulesContent = `ebtable_nat 16384 1 - Live 0x0000000000000000
ebtable_broute 16384 1 - Live 0x0000000000000000
bridge 172032 1 ebtable_broute, Live 0x0000000000000000
ip6table_nat 16384 1 - Live 0x0000000000000000
nf_nat_ipv6 16384 1 ip6table_nat, Live 0x0000000000000000
ip6table_mangle 16384 1 - Live 0x0000000000000000
ip6table_raw 16384 1 - Live 0x0000000000000000
ip6table_security 16384 1 - Live 0x0000000000000000
iptable_nat 16384 1 - Live 0x0000000000000000
nf_nat_ipv4 16384 1 iptable_nat, Live 0x0000000000000000
iptable_mangle 16384 1 - Live 0x0000000000000000
iptable_raw 16384 1 - Live 0x0000000000000000
iptable_security 16384 1 - Live 0x0000000000000000
ebtable_filter 16384 1 - Live 0x0000000000000000
ebtables 36864 3 ebtable_nat,ebtable_broute,ebtable_filter, Live 0x0000000000000000
ip6table_filter 16384 1 - Live 0x0000000000000000
ip6_tables 28672 5 ip6table_nat,ip6table_mangle,ip6table_raw,ip6table_security,ip6table_filter, Live 0x0000000000000000
iptable_filter 16384 1 - Live 0x0000000000000000
ip_tables 28672 5 iptable_nat,iptable_mangle,iptable_raw,iptable_security,iptable_filter, Live 0x0000000000000000
x_tables 40960 23 xt_multiport,xt_nat,xt_addrtype,xt_mark,xt_comment,xt_CHECKSUM,ipt_MASQUERADE,xt_tcpudp,ip6t_rpfilter,ip6t_REJECT,ipt_REJECT,xt_conntrack,ip6table_mangle,ip6table_raw,ip6table_security,iptable_mangle,iptable_raw,iptable_security,ebtables,ip6table_filter,ip6_tables,iptable_filter,ip_tables, Live 0x0000000000000000`
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) {
	TestingT(t)
}

type ModulesTestSuite struct{}

var _ = Suite(&ModulesTestSuite{})

func (s *ModulesTestSuite) TestInit(c *C) {
	var manager ModulesManager
	c.Assert(manager.Init(), IsNil)
}

func (s *ModulesTestSuite) TestFindModules(c *C) {
	manager := &ModulesManager{
		modulesList: []string{
			"ip6_tables",
			"ip6table_mangle",
			"ip6table_filter",
			"ip6table_security",
			"ip6table_raw",
			"ip6table_nat",
		},
	}
	testCases := []struct {
		modulesToFind []string
		isSubset      bool
		expectedDiff  []string
	}{
		{
			modulesToFind: []string{
				"ip6_tables",
				"ip6table_mangle",
				"ip6table_filter",
				"ip6table_security",
				"ip6table_raw",
				"ip6table_nat",
			},
			isSubset:     true,
			expectedDiff: nil,
		},
		{
			modulesToFind: []string{
				"ip6_tables",
				"ip6table_mangle",
				"ip6table_raw",
			},
			isSubset:     true,
			expectedDiff: nil,
		},
		{
			modulesToFind: []string{
				"ip6_tables",
				"ip6table_mangle",
				"ip6table_raw",
				"foo_module",
			},
			isSubset:     false,
			expectedDiff: []string{"foo_module"},
		},
		{
			modulesToFind: []string{
				"foo_module",
				"bar_module",
			},
			isSubset:     false,
			expectedDiff: []string{"foo_module", "bar_module"},
		},
	}
	for _, tc := range testCases {
		found, diff := manager.FindModules(tc.modulesToFind...)
		c.Assert(found, Equals, tc.isSubset)
		c.Assert(diff, checker.DeepEquals, tc.expectedDiff)
	}
}

func (s *ModulesTestSuite) TestParseModuleFile(c *C) {
	expectedLength := 20
	expectedModules := []string{
		"ebtable_nat",
		"ebtable_broute",
		"bridge",
		"ip6table_nat",
		"nf_nat_ipv6",
		"ip6table_mangle",
		"ip6table_raw",
		"ip6table_security",
		"iptable_nat",
		"nf_nat_ipv4",
		"iptable_mangle",
		"iptable_raw",
		"iptable_security",
		"ebtable_filter",
		"ebtables",
		"ip6table_filter",
		"ip6_tables",
		"iptable_filter",
		"ip_tables",
		"x_tables",
	}

	r := bytes.NewBuffer([]byte(modulesContent))
	moduleInfos, err := parseModulesFile(r)
	c.Assert(err, IsNil)
	c.Assert(moduleInfos, HasLen, expectedLength)
	c.Assert(moduleInfos, checker.DeepEquals, expectedModules)
}

func (s *ModulesTestSuite) TestListModules(c *C) {
	_, err := listModules()
	c.Assert(err, IsNil)
}
<|endoftext|>


<commit_before>// Copyright 2016-2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package manager

import (
	"math"
	"time"

	"github.com/cilium/cilium/pkg/datapath"
	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/metrics"
	"github.com/cilium/cilium/pkg/node"

	"github.com/prometheus/client_golang/prometheus"
)

var (
	baseBackgroundSyncInterval = time.Minute
)

type nodeEntry struct {
	// mutex serves two purposes:
	// 1. Serialize any direct access to the node field in this entry.
	// 2. Serialize all calls do the datapath layer for a particular node.
	//
	// See description of Manager.mutex for more details
	//
	// If both the nodeEntry.mutex and Manager.mutex must be held, then the
	// Manager.mutex must *always* be acquired first.
	mutex lock.Mutex
	node  node.Node
}

// Manager is the entity that manages a collection of nodes
type Manager struct {
	// mutex is the lock protecting access to the nodes map. The mutex must
	// be held for any access of the nodes map.
	//
	// The manager mutex works together with the entry mutex in the
	// following way to minimize the duration the manager mutex is held:
	//
	// 1. Acquire manager mutex to safely access nodes map and to retrieve
	//    node entry.
	// 2. Acquire mutex of the entry while the manager mutex is still held.
	//    This guarantees that no change to the entry has happened.
	// 3. Release of the manager mutex to unblock changes or reads to other
	//    node entries.
	// 4. Change of entry data or performing of datapath interactions
	// 5. Release of the entry mutex
	//
	// If both the nodeEntry.mutex and Manager.mutex must be held, then the
	// Manager.mutex must *always* be acquired first.
	mutex lock.RWMutex

	// nodes is the list of nodes. Access must be protected via mutex.
	nodes map[node.Identity]*nodeEntry

	// datapath is the interface responsible for this node manager
	datapath datapath.NodeHandler

	// closeChan is closed when the manager is closed
	closeChan chan struct{}

	// name is the name of the manager. It must be unique and feasibility
	// to be used a prometheus metric name.
	name string

	// metricEventsReceived is the prometheus metric to track the number of
	// node events received
	metricEventsReceived *prometheus.CounterVec

	// metricNumNodes is the prometheus metric to track the number of nodes
	// being managed
	metricNumNodes prometheus.Gauge

	// metricDatapathValidations is the prometheus metric to track the
	// number of datapath node validation calls
	metricDatapathValidations prometheus.Counter
}

// NewManager returns a new node manager
func NewManager(name string, datapath datapath.NodeHandler) (*Manager, error) {
	m := &Manager{
		name:      name,
		nodes:     map[node.Identity]*nodeEntry{},
		datapath:  datapath,
		closeChan: make(chan struct{}),
	}

	m.metricEventsReceived = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: metrics.Namespace,
		Subsystem: "nodes",
		Name:      name + "_events_received_total",
		Help:      "Number of node events received",
	}, []string{"eventType", "source"})

	m.metricNumNodes = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: metrics.Namespace,
		Subsystem: "nodes",
		Name:      name + "_num",
		Help:      "Number of nodes managed",
	})

	m.metricDatapathValidations = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: metrics.Namespace,
		Subsystem: "nodes",
		Name:      name + "_datapath_validations_total",
		Help:      "Number of validation calls to implement the datapath implemention of a node",
	})

	err := metrics.RegisterList([]prometheus.Collector{m.metricDatapathValidations, m.metricEventsReceived, m.metricNumNodes})
	if err != nil {
		return nil, err
	}

	go m.backgroundSync()

	return m, nil
}

// Close shuts down a node manager
func (m *Manager) Close() {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	close(m.closeChan)

	metrics.Unregister(m.metricNumNodes)
	metrics.Unregister(m.metricEventsReceived)
	metrics.Unregister(m.metricDatapathValidations)

	// delete all nodes to clean up the datapath for each node
	for _, n := range m.nodes {
		n.mutex.Lock()
		m.datapath.NodeDelete(n.node)
		n.mutex.Unlock()
	}
}

// ClusterSizeDependantInterval returns a time.Duration that is dependant on
// the cluster size, i.e. the number of nodes that have been discovered. This
// can be used to control sync intervals of shared or centralized resources to
// avoid overloading these resources as the cluster grows.
//
// Example sync interval with baseInterval = 1 * time.Minute
//
// nodes | sync interval
// ------+-----------------
// 1     |   41.588830833s
// 2     | 1m05.916737320s
// 4     | 1m36.566274746s
// 8     | 2m11.833474640s
// 16    | 2m49.992800643s
// 32    | 3m29.790453687s
// 64    | 4m10.463236193s
// 128   | 4m51.588744261s
// 256   | 5m32.944565093s
// 512   | 6m14.416550710s
// 1024  | 6m55.946873494s
// 2048  | 7m37.506428894s
// 4096  | 8m19.080616652s
// 8192  | 9m00.662124608s
// 16384 | 9m42.247293667s
func (m *Manager) ClusterSizeDependantInterval(baseInterval time.Duration) time.Duration {
	m.mutex.RLock()
	numNodes := len(m.nodes)
	m.mutex.RUnlock()

	// no nodes are being managed, no work will be performed, return
	// baseInterval to check again in a reasonable timeframe
	if numNodes == 0 {
		return baseInterval
	}

	waitNanoseconds := float64(baseInterval.Nanoseconds()) * math.Log1p(float64(numNodes))
	return time.Duration(int64(waitNanoseconds))

}

func (m *Manager) backgroundSyncInterval() time.Duration {
	return m.ClusterSizeDependantInterval(baseBackgroundSyncInterval)
}

func (m *Manager) backgroundSync() {
	for {
		syncInterval := m.backgroundSyncInterval()
		log.WithField("syncInterval", syncInterval.String()).Debug("Performing regular background work")

		// get a copy of the node identities to avoid locking the entire manager
		// throughout the process of running the datapath validation.
		nodes := m.GetNodeIdentities()
		for _, nodeIdentity := range nodes {
			// Retrieve latest node information in case any event
			// changed the node since the call to GetNodes()
			m.mutex.RLock()
			entry, ok := m.nodes[nodeIdentity]
			if !ok {
				m.mutex.RUnlock()
				continue
			}

			entry.mutex.Lock()
			m.mutex.RUnlock()
			m.datapath.NodeValidateImplementation(entry.node)
			entry.mutex.Unlock()

			m.metricDatapathValidations.Inc()
		}

		select {
		case <-m.closeChan:
			return
		case <-time.After(syncInterval):
		}
	}
}

// overwriteAllowed returns true if an update from newSource can overwrite a node owned by oldSource.
func overwriteAllowed(oldSource, newSource node.Source) bool {
	switch newSource {
	// the local node always takes precedence
	case node.FromLocalNode:
		return true

	// agent local updates can overwrite everything except for the local
	// node
	case node.FromAgentLocal:
		return oldSource != node.FromLocalNode

	// kvstore updates can overwrite everything except agent local updates and local node
	case node.FromKVStore:
		return oldSource != node.FromAgentLocal && oldSource != node.FromLocalNode

	// kubernetes updates can only overwrite kubernetes nodes
	case node.FromKubernetes:
		return oldSource != node.FromAgentLocal && oldSource != node.FromLocalNode && oldSource != node.FromKVStore

	default:
		return false
	}
}

// NodeSoftUpdated is called after the information of a node has be upated but
// unlike a NodeUpdated does not require the datapath to be updated.
func (m *Manager) NodeSoftUpdated(n node.Node) {
	log.Debugf("Received soft node update event from %s: %#v", n.Source, n)
	m.nodeUpdated(n, false)
}

// NodeUpdated is called after the information of a node has been updated. The
// node in the manager is added or updated if the source is allowed to update
// the node. If an update or addition has occurred, NodeUpdate() of the datapath
// interface is invoked.
func (m *Manager) NodeUpdated(n node.Node) {
	log.Debugf("Received node update event from %s: %#v", n.Source, n)
	m.nodeUpdated(n, true)
}

func (m *Manager) nodeUpdated(n node.Node, dpUpdate bool) {
	nodeIdentity := n.Identity()

	m.mutex.Lock()
	entry, oldNodeExists := m.nodes[nodeIdentity]
	if oldNodeExists {
		m.metricEventsReceived.WithLabelValues("update", string(n.Source)).Inc()

		if !overwriteAllowed(entry.node.Source, n.Source) {
			m.mutex.Unlock()
			return
		}

		entry.mutex.Lock()
		m.mutex.Unlock()
		oldNode := entry.node
		entry.node = n
		if dpUpdate {
			m.datapath.NodeUpdate(oldNode, entry.node)
		}
		entry.mutex.Unlock()
	} else {
		m.metricEventsReceived.WithLabelValues("add", string(n.Source)).Inc()
		m.metricNumNodes.Inc()

		entry = &nodeEntry{node: n}
		entry.mutex.Lock()
		m.nodes[nodeIdentity] = entry
		m.mutex.Unlock()
		if dpUpdate {
			m.datapath.NodeAdd(entry.node)
		}
		entry.mutex.Unlock()
	}
}

// NodeDeleted is called after a node has been deleted. It removes the node
// from the manager if the node is still owned by the source of which the event
// orgins from. If the node was removed, NodeDelete() is invoked of the
// datapath interface.
func (m *Manager) NodeDeleted(n node.Node) {
	m.metricEventsReceived.WithLabelValues("delete", string(n.Source)).Inc()

	log.Debugf("Received node delete event from %s", n.Source)

	nodeIdentity := n.Identity()

	m.mutex.Lock()
	entry, oldNodeExists := m.nodes[nodeIdentity]
	if !oldNodeExists {
		m.mutex.Unlock()
		return
	}

	// If the source is Kubernetes and the node is the node we are running on
	// Kubernetes is giving us a hint it is about to delete our node. Close down
	// the agent gracefully in this case.
	if n.Source != entry.node.Source {
		m.mutex.Unlock()
		if n.IsLocal() && n.Source == node.FromKubernetes {
			log.Debugf("Kubernetes is deleting local node, close manager")
			m.Close()
		} else {
			log.Debugf("Ignoring delete event of node %s from source %s. The node is owned by %s",
				n.Name, n.Source, entry.node.Source)
		}
		return
	}

	m.metricNumNodes.Dec()

	entry.mutex.Lock()
	delete(m.nodes, nodeIdentity)
	m.mutex.Unlock()
	m.datapath.NodeDelete(n)
	entry.mutex.Unlock()
}

// Exists returns true if a node with the name exists
func (m *Manager) Exists(id node.Identity) bool {
	m.mutex.RLock()
	defer m.mutex.RUnlock()
	_, ok := m.nodes[id]
	return ok
}

// GetNodeIdentities returns a list of all node identities store in node
// manager.
func (m *Manager) GetNodeIdentities() []node.Identity {
	m.mutex.RLock()
	defer m.mutex.RUnlock()

	nodes := make([]node.Identity, 0, len(m.nodes))
	for nodeIdentity := range m.nodes {
		nodes = append(nodes, nodeIdentity)
	}

	return nodes
}

// GetNodes returns a copy of all of the nodes as a map from Identity to Node.
func (m *Manager) GetNodes() map[node.Identity]node.Node {
	m.mutex.RLock()
	defer m.mutex.RUnlock()

	nodes := make(map[node.Identity]node.Node)
	for nodeIdentity, entry := range m.nodes {
		entry.mutex.Lock()
		nodes[nodeIdentity] = entry.node
		entry.mutex.Unlock()
	}

	return nodes
}

// DeleteAllNodes deletes all nodes from the node maanger.
func (m *Manager) DeleteAllNodes() {
	m.mutex.Lock()
	for _, entry := range m.nodes {
		entry.mutex.Lock()
		m.datapath.NodeDelete(entry.node)
		entry.mutex.Unlock()
	}
	m.nodes = map[node.Identity]*nodeEntry{}
	m.mutex.Unlock()
}
<commit_msg>node/manager: add a subscription event based mechanism for node events<commit_after>// Copyright 2016-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package manager

import (
	"math"
	"time"

	"github.com/cilium/cilium/pkg/datapath"
	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/metrics"
	"github.com/cilium/cilium/pkg/node"

	"github.com/prometheus/client_golang/prometheus"
)

var (
	baseBackgroundSyncInterval = time.Minute
)

type nodeEntry struct {
	// mutex serves two purposes:
	// 1. Serialize any direct access to the node field in this entry.
	// 2. Serialize all calls do the datapath layer for a particular node.
	//
	// See description of Manager.mutex for more details
	//
	// If both the nodeEntry.mutex and Manager.mutex must be held, then the
	// Manager.mutex must *always* be acquired first.
	mutex lock.Mutex
	node  node.Node
}

// Manager is the entity that manages a collection of nodes
type Manager struct {
	// mutex is the lock protecting access to the nodes map. The mutex must
	// be held for any access of the nodes map.
	//
	// The manager mutex works together with the entry mutex in the
	// following way to minimize the duration the manager mutex is held:
	//
	// 1. Acquire manager mutex to safely access nodes map and to retrieve
	//    node entry.
	// 2. Acquire mutex of the entry while the manager mutex is still held.
	//    This guarantees that no change to the entry has happened.
	// 3. Release of the manager mutex to unblock changes or reads to other
	//    node entries.
	// 4. Change of entry data or performing of datapath interactions
	// 5. Release of the entry mutex
	//
	// If both the nodeEntry.mutex and Manager.mutex must be held, then the
	// Manager.mutex must *always* be acquired first.
	mutex lock.RWMutex

	// nodes is the list of nodes. Access must be protected via mutex.
	nodes map[node.Identity]*nodeEntry

	// nodeHandlersMu protects the nodeHandlers map against concurrent access.
	nodeHandlersMu lock.RWMutex
	// nodeHandlers has a slice containing all node handlers subscribed to node
	// events.
	nodeHandlers map[datapath.NodeHandler]struct{}

	// closeChan is closed when the manager is closed
	closeChan chan struct{}

	// name is the name of the manager. It must be unique and feasibility
	// to be used a prometheus metric name.
	name string

	// metricEventsReceived is the prometheus metric to track the number of
	// node events received
	metricEventsReceived *prometheus.CounterVec

	// metricNumNodes is the prometheus metric to track the number of nodes
	// being managed
	metricNumNodes prometheus.Gauge

	// metricDatapathValidations is the prometheus metric to track the
	// number of datapath node validation calls
	metricDatapathValidations prometheus.Counter
}

// Subscribe subscribes the given node handler to node events.
func (m *Manager) Subscribe(nh datapath.NodeHandler) {
	m.nodeHandlersMu.Lock()
	m.nodeHandlers[nh] = struct{}{}
	m.nodeHandlersMu.Unlock()
	// Add all nodes already received by the manager.
	for _, v := range m.nodes {
		v.mutex.Lock()
		nh.NodeAdd(v.node)
		v.mutex.Unlock()
	}
}

// Unsubscribe unsubscribes the given node handler with node events.
func (m *Manager) Unsubscribe(nh datapath.NodeHandler) {
	m.nodeHandlersMu.Lock()
	delete(m.nodeHandlers, nh)
	m.nodeHandlersMu.Unlock()
}

// Iter executes the given function in all subscribed node handlers.
func (m *Manager) Iter(f func(nh datapath.NodeHandler)) {
	m.nodeHandlersMu.RLock()
	defer m.nodeHandlersMu.RUnlock()

	for nh := range m.nodeHandlers {
		f(nh)
	}
}

// NewManager returns a new node manager
func NewManager(name string, dp datapath.NodeHandler) (*Manager, error) {
	m := &Manager{
		name:         name,
		nodes:        map[node.Identity]*nodeEntry{},
		nodeHandlers: map[datapath.NodeHandler]struct{}{},
		closeChan:    make(chan struct{}),
	}
	m.Subscribe(dp)

	m.metricEventsReceived = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: metrics.Namespace,
		Subsystem: "nodes",
		Name:      name + "_events_received_total",
		Help:      "Number of node events received",
	}, []string{"eventType", "source"})

	m.metricNumNodes = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: metrics.Namespace,
		Subsystem: "nodes",
		Name:      name + "_num",
		Help:      "Number of nodes managed",
	})

	m.metricDatapathValidations = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: metrics.Namespace,
		Subsystem: "nodes",
		Name:      name + "_datapath_validations_total",
		Help:      "Number of validation calls to implement the datapath implemention of a node",
	})

	err := metrics.RegisterList([]prometheus.Collector{m.metricDatapathValidations, m.metricEventsReceived, m.metricNumNodes})
	if err != nil {
		return nil, err
	}

	go m.backgroundSync()

	return m, nil
}

// Close shuts down a node manager
func (m *Manager) Close() {
	m.mutex.Lock()
	defer m.mutex.Unlock()

	close(m.closeChan)

	metrics.Unregister(m.metricNumNodes)
	metrics.Unregister(m.metricEventsReceived)
	metrics.Unregister(m.metricDatapathValidations)

	// delete all nodes to clean up the datapath for each node
	for _, n := range m.nodes {
		n.mutex.Lock()
		m.Iter(func(nh datapath.NodeHandler) {
			nh.NodeDelete(n.node)
		})
		n.mutex.Unlock()
	}
}

// ClusterSizeDependantInterval returns a time.Duration that is dependant on
// the cluster size, i.e. the number of nodes that have been discovered. This
// can be used to control sync intervals of shared or centralized resources to
// avoid overloading these resources as the cluster grows.
//
// Example sync interval with baseInterval = 1 * time.Minute
//
// nodes | sync interval
// ------+-----------------
// 1     |   41.588830833s
// 2     | 1m05.916737320s
// 4     | 1m36.566274746s
// 8     | 2m11.833474640s
// 16    | 2m49.992800643s
// 32    | 3m29.790453687s
// 64    | 4m10.463236193s
// 128   | 4m51.588744261s
// 256   | 5m32.944565093s
// 512   | 6m14.416550710s
// 1024  | 6m55.946873494s
// 2048  | 7m37.506428894s
// 4096  | 8m19.080616652s
// 8192  | 9m00.662124608s
// 16384 | 9m42.247293667s
func (m *Manager) ClusterSizeDependantInterval(baseInterval time.Duration) time.Duration {
	m.mutex.RLock()
	numNodes := len(m.nodes)
	m.mutex.RUnlock()

	// no nodes are being managed, no work will be performed, return
	// baseInterval to check again in a reasonable timeframe
	if numNodes == 0 {
		return baseInterval
	}

	waitNanoseconds := float64(baseInterval.Nanoseconds()) * math.Log1p(float64(numNodes))
	return time.Duration(int64(waitNanoseconds))

}

func (m *Manager) backgroundSyncInterval() time.Duration {
	return m.ClusterSizeDependantInterval(baseBackgroundSyncInterval)
}

func (m *Manager) backgroundSync() {
	for {
		syncInterval := m.backgroundSyncInterval()
		log.WithField("syncInterval", syncInterval.String()).Debug("Performing regular background work")

		// get a copy of the node identities to avoid locking the entire manager
		// throughout the process of running the datapath validation.
		nodes := m.GetNodeIdentities()
		for _, nodeIdentity := range nodes {
			// Retrieve latest node information in case any event
			// changed the node since the call to GetNodes()
			m.mutex.RLock()
			entry, ok := m.nodes[nodeIdentity]
			if !ok {
				m.mutex.RUnlock()
				continue
			}

			entry.mutex.Lock()
			m.mutex.RUnlock()
			m.Iter(func(nh datapath.NodeHandler) {
				nh.NodeValidateImplementation(entry.node)
			})
			entry.mutex.Unlock()

			m.metricDatapathValidations.Inc()
		}

		select {
		case <-m.closeChan:
			return
		case <-time.After(syncInterval):
		}
	}
}

// overwriteAllowed returns true if an update from newSource can overwrite a node owned by oldSource.
func overwriteAllowed(oldSource, newSource node.Source) bool {
	switch newSource {
	// the local node always takes precedence
	case node.FromLocalNode:
		return true

	// agent local updates can overwrite everything except for the local
	// node
	case node.FromAgentLocal:
		return oldSource != node.FromLocalNode

	// kvstore updates can overwrite everything except agent local updates and local node
	case node.FromKVStore:
		return oldSource != node.FromAgentLocal && oldSource != node.FromLocalNode

	// kubernetes updates can only overwrite kubernetes nodes
	case node.FromKubernetes:
		return oldSource != node.FromAgentLocal && oldSource != node.FromLocalNode && oldSource != node.FromKVStore

	default:
		return false
	}
}

// NodeSoftUpdated is called after the information of a node has be upated but
// unlike a NodeUpdated does not require the datapath to be updated.
func (m *Manager) NodeSoftUpdated(n node.Node) {
	log.Debugf("Received soft node update event from %s: %#v", n.Source, n)
	m.nodeUpdated(n, false)
}

// NodeUpdated is called after the information of a node has been updated. The
// node in the manager is added or updated if the source is allowed to update
// the node. If an update or addition has occurred, NodeUpdate() of the datapath
// interface is invoked.
func (m *Manager) NodeUpdated(n node.Node) {
	log.Debugf("Received node update event from %s: %#v", n.Source, n)
	m.nodeUpdated(n, true)
}

func (m *Manager) nodeUpdated(n node.Node, dpUpdate bool) {
	nodeIdentity := n.Identity()

	m.mutex.Lock()
	entry, oldNodeExists := m.nodes[nodeIdentity]
	if oldNodeExists {
		m.metricEventsReceived.WithLabelValues("update", string(n.Source)).Inc()

		if !overwriteAllowed(entry.node.Source, n.Source) {
			m.mutex.Unlock()
			return
		}

		entry.mutex.Lock()
		m.mutex.Unlock()
		oldNode := entry.node
		entry.node = n
		if dpUpdate {
			m.Iter(func(nh datapath.NodeHandler) {
				nh.NodeUpdate(oldNode, entry.node)
			})
		}
		entry.mutex.Unlock()
	} else {
		m.metricEventsReceived.WithLabelValues("add", string(n.Source)).Inc()
		m.metricNumNodes.Inc()

		entry = &nodeEntry{node: n}
		entry.mutex.Lock()
		m.nodes[nodeIdentity] = entry
		m.mutex.Unlock()
		if dpUpdate {
			m.Iter(func(nh datapath.NodeHandler) {
				nh.NodeAdd(entry.node)
			})
		}
		entry.mutex.Unlock()
	}
}

// NodeDeleted is called after a node has been deleted. It removes the node
// from the manager if the node is still owned by the source of which the event
// orgins from. If the node was removed, NodeDelete() is invoked of the
// datapath interface.
func (m *Manager) NodeDeleted(n node.Node) {
	m.metricEventsReceived.WithLabelValues("delete", string(n.Source)).Inc()

	log.Debugf("Received node delete event from %s", n.Source)

	nodeIdentity := n.Identity()

	m.mutex.Lock()
	entry, oldNodeExists := m.nodes[nodeIdentity]
	if !oldNodeExists {
		m.mutex.Unlock()
		return
	}

	// If the source is Kubernetes and the node is the node we are running on
	// Kubernetes is giving us a hint it is about to delete our node. Close down
	// the agent gracefully in this case.
	if n.Source != entry.node.Source {
		m.mutex.Unlock()
		if n.IsLocal() && n.Source == node.FromKubernetes {
			log.Debugf("Kubernetes is deleting local node, close manager")
			m.Close()
		} else {
			log.Debugf("Ignoring delete event of node %s from source %s. The node is owned by %s",
				n.Name, n.Source, entry.node.Source)
		}
		return
	}

	m.metricNumNodes.Dec()

	entry.mutex.Lock()
	delete(m.nodes, nodeIdentity)
	m.mutex.Unlock()
	m.Iter(func(nh datapath.NodeHandler) {
		nh.NodeDelete(n)
	})
	entry.mutex.Unlock()
}

// Exists returns true if a node with the name exists
func (m *Manager) Exists(id node.Identity) bool {
	m.mutex.RLock()
	defer m.mutex.RUnlock()
	_, ok := m.nodes[id]
	return ok
}

// GetNodeIdentities returns a list of all node identities store in node
// manager.
func (m *Manager) GetNodeIdentities() []node.Identity {
	m.mutex.RLock()
	defer m.mutex.RUnlock()

	nodes := make([]node.Identity, 0, len(m.nodes))
	for nodeIdentity := range m.nodes {
		nodes = append(nodes, nodeIdentity)
	}

	return nodes
}

// GetNodes returns a copy of all of the nodes as a map from Identity to Node.
func (m *Manager) GetNodes() map[node.Identity]node.Node {
	m.mutex.RLock()
	defer m.mutex.RUnlock()

	nodes := make(map[node.Identity]node.Node)
	for nodeIdentity, entry := range m.nodes {
		entry.mutex.Lock()
		nodes[nodeIdentity] = entry.node
		entry.mutex.Unlock()
	}

	return nodes
}

// DeleteAllNodes deletes all nodes from the node maanger.
func (m *Manager) DeleteAllNodes() {
	m.mutex.Lock()
	for _, entry := range m.nodes {
		entry.mutex.Lock()
		m.Iter(func(nh datapath.NodeHandler) {
			nh.NodeDelete(entry.node)
		})
		entry.mutex.Unlock()
	}
	m.nodes = map[node.Identity]*nodeEntry{}
	m.mutex.Unlock()
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
	"math"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/golang/glog"
)

// the unused capacity is calculated on a scale of 0-10
// 0 being the lowest priority and 10 being the highest
func calculateScore(requested, capacity int64, node string) int {
	if capacity == 0 {
		return 0
	}
	if requested > capacity {
		glog.Errorf("Combined requested resources from existing pods exceeds capacity on minion: %s", node)
		return 0
	}
	return int(((capacity - requested) * 10) / capacity)
}

// Calculate the occupancy on a node.  'node' has information about the resources on the node.
// 'pods' is a list of pods currently scheduled on the node.
func calculateOccupancy(pod api.Pod, node api.Node, pods []api.Pod) HostPriority {
	totalMilliCPU := int64(0)
	totalMemory := int64(0)
	for _, existingPod := range pods {
		for _, container := range existingPod.Spec.Containers {
			totalMilliCPU += container.Resources.Limits.Cpu().MilliValue()
			totalMemory += container.Resources.Limits.Memory().Value()
		}
	}
	// Add the resources requested by the current pod being scheduled.
	// This also helps differentiate between differently sized, but empty, minions.
	for _, container := range pod.Spec.Containers {
		totalMilliCPU += container.Resources.Limits.Cpu().MilliValue()
		totalMemory += container.Resources.Limits.Memory().Value()
	}

	capacityMilliCPU := node.Status.Capacity.Cpu().MilliValue()
	capacityMemory := node.Status.Capacity.Memory().Value()

	cpuScore := calculateScore(totalMilliCPU, capacityMilliCPU, node.Name)
	memoryScore := calculateScore(totalMemory, capacityMemory, node.Name)
	glog.V(4).Infof(
		"%v -> %v: Least Requested Priority, AbsoluteRequested: (%d, %d) / (%d, %d) Score: (%d, %d)",
		pod.Name, node.Name,
		totalMilliCPU, totalMemory,
		capacityMilliCPU, capacityMemory,
		cpuScore, memoryScore,
	)

	return HostPriority{
		host:  node.Name,
		score: int((cpuScore + memoryScore) / 2),
	}
}

// LeastRequestedPriority is a priority function that favors nodes with fewer requested resources.
// It calculates the percentage of memory and CPU requested by pods scheduled on the node, and prioritizes
// based on the minimum of the average of the fraction of requested to capacity.
// Details: (Sum(requested cpu) / Capacity + Sum(requested memory) / Capacity) * 50
func LeastRequestedPriority(pod api.Pod, podLister PodLister, minionLister MinionLister) (HostPriorityList, error) {
	nodes, err := minionLister.List()
	if err != nil {
		return HostPriorityList{}, err
	}
	podsToMachines, err := MapPodsToMachines(podLister)

	list := HostPriorityList{}
	for _, node := range nodes.Items {
		list = append(list, calculateOccupancy(pod, node, podsToMachines[node.Name]))
	}
	return list, nil
}

type NodeLabelPrioritizer struct {
	label    string
	presence bool
}

func NewNodeLabelPriority(label string, presence bool) PriorityFunction {
	labelPrioritizer := &NodeLabelPrioritizer{
		label:    label,
		presence: presence,
	}
	return labelPrioritizer.CalculateNodeLabelPriority
}

// CalculateNodeLabelPriority checks whether a particular label exists on a minion or not, regardless of its value.
// If presence is true, prioritizes minions that have the specified label, regardless of value.
// If presence is false, prioritizes minions that do not have the specified label.
func (n *NodeLabelPrioritizer) CalculateNodeLabelPriority(pod api.Pod, podLister PodLister, minionLister MinionLister) (HostPriorityList, error) {
	var score int
	minions, err := minionLister.List()
	if err != nil {
		return nil, err
	}

	labeledMinions := map[string]bool{}
	for _, minion := range minions.Items {
		exists := labels.Set(minion.Labels).Has(n.label)
		labeledMinions[minion.Name] = (exists && n.presence) || (!exists && !n.presence)
	}

	result := []HostPriority{}
	//score int - scale of 0-10
	// 0 being the lowest priority and 10 being the highest
	for minionName, success := range labeledMinions {
		if success {
			score = 10
		} else {
			score = 0
		}
		result = append(result, HostPriority{host: minionName, score: score})
	}
	return result, nil
}

// BalancedResourceAllocation favors nodes with balanced resource usage rate.
// BalancedResourceAllocation should **NOT** be used alone, and **MUST** be used together with LeastRequestedPriority.
// It calculates the difference between the cpu and memory fracion of capacity, and prioritizes the host based on how
// close the two metrics are to each other.
// Detail: score = 10 - abs(cpuFraction-memoryFraction)*10. The algorithm is partly inspired by:
// "Wei Huang et al. An Energy Efficient Virtual Machine Placement Algorithm with Balanced Resource Utilization"
func BalancedResourceAllocation(pod api.Pod, podLister PodLister, minionLister MinionLister) (HostPriorityList, error) {
	nodes, err := minionLister.List()
	if err != nil {
		return HostPriorityList{}, err
	}
	podsToMachines, err := MapPodsToMachines(podLister)

	list := HostPriorityList{}
	for _, node := range nodes.Items {
		list = append(list, calculateBalancedResourceAllocation(pod, node, podsToMachines[node.Name]))
	}
	return list, nil
}

func calculateBalancedResourceAllocation(pod api.Pod, node api.Node, pods []api.Pod) HostPriority {
	totalMilliCPU := int64(0)
	totalMemory := int64(0)
	score := int(0)
	for _, existingPod := range pods {
		for _, container := range existingPod.Spec.Containers {
			totalMilliCPU += container.Resources.Limits.Cpu().MilliValue()
			totalMemory += container.Resources.Limits.Memory().Value()
		}
	}
	// Add the resources requested by the current pod being scheduled.
	// This also helps differentiate between differently sized, but empty, minions.
	for _, container := range pod.Spec.Containers {
		totalMilliCPU += container.Resources.Limits.Cpu().MilliValue()
		totalMemory += container.Resources.Limits.Memory().Value()
	}

	capacityMilliCPU := node.Status.Capacity.Cpu().MilliValue()
	capacityMemory := node.Status.Capacity.Memory().Value()

	cpuFraction := fractionOfCapacity(totalMilliCPU, capacityMilliCPU, node.Name)
	memoryFraction := fractionOfCapacity(totalMemory, capacityMemory, node.Name)
	if cpuFraction >= 1 || memoryFraction >= 1 {
		// if requested >= capacity, the corresponding host should never be preferrred.
		score = 0
	} else {
		// Upper and lower boundary of difference between cpuFraction and memoryFraction are -1 and 1
		// respectively. Multilying the absolute value of the difference by 10 scales the value to
		// 0-10 with 0 representing well balanced allocation and 10 poorly balanced. Subtracting it from
		// 10 leads to the score which also scales from 0 to 10 while 10 representing well balanced.
		diff := math.Abs(cpuFraction - memoryFraction)
		score = int(10 - diff*10)
	}
	glog.V(4).Infof(
		"%v -> %v: Balanced Resource Allocation, Absolute/Requested: (%d, %d) / (%d, %d) Score: (%d)",
		pod.Name, node.Name,
		totalMilliCPU, totalMemory,
		capacityMilliCPU, capacityMemory,
		score,
	)

	return HostPriority{
		host:  node.Name,
		score: score,
	}
}

func fractionOfCapacity(requested, capacity int64, node string) float64 {
	if capacity == 0 {
		return 1
	}
	return float64(requested) / float64(capacity)
}
<commit_msg>Update priorities.go<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
	"math"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/golang/glog"
)

// the unused capacity is calculated on a scale of 0-10
// 0 being the lowest priority and 10 being the highest
func calculateScore(requested, capacity int64, node string) int {
	if capacity == 0 {
		return 0
	}
	if requested > capacity {
		glog.Infof("Combined requested resources from existing pods exceeds capacity on minion: %s", node)
		return 0
	}
	return int(((capacity - requested) * 10) / capacity)
}

// Calculate the occupancy on a node.  'node' has information about the resources on the node.
// 'pods' is a list of pods currently scheduled on the node.
func calculateOccupancy(pod api.Pod, node api.Node, pods []api.Pod) HostPriority {
	totalMilliCPU := int64(0)
	totalMemory := int64(0)
	for _, existingPod := range pods {
		for _, container := range existingPod.Spec.Containers {
			totalMilliCPU += container.Resources.Limits.Cpu().MilliValue()
			totalMemory += container.Resources.Limits.Memory().Value()
		}
	}
	// Add the resources requested by the current pod being scheduled.
	// This also helps differentiate between differently sized, but empty, minions.
	for _, container := range pod.Spec.Containers {
		totalMilliCPU += container.Resources.Limits.Cpu().MilliValue()
		totalMemory += container.Resources.Limits.Memory().Value()
	}

	capacityMilliCPU := node.Status.Capacity.Cpu().MilliValue()
	capacityMemory := node.Status.Capacity.Memory().Value()

	cpuScore := calculateScore(totalMilliCPU, capacityMilliCPU, node.Name)
	memoryScore := calculateScore(totalMemory, capacityMemory, node.Name)
	glog.V(4).Infof(
		"%v -> %v: Least Requested Priority, Absolute/Requested: (%d, %d) / (%d, %d) Score: (%d, %d)",
		pod.Name, node.Name,
		totalMilliCPU, totalMemory,
		capacityMilliCPU, capacityMemory,
		cpuScore, memoryScore,
	)

	return HostPriority{
		host:  node.Name,
		score: int((cpuScore + memoryScore) / 2),
	}
}

// LeastRequestedPriority is a priority function that favors nodes with fewer requested resources.
// It calculates the percentage of memory and CPU requested by pods scheduled on the node, and prioritizes
// based on the minimum of the average of the fraction of requested to capacity.
// Details: (Sum(requested cpu) / Capacity + Sum(requested memory) / Capacity) * 50
func LeastRequestedPriority(pod api.Pod, podLister PodLister, minionLister MinionLister) (HostPriorityList, error) {
	nodes, err := minionLister.List()
	if err != nil {
		return HostPriorityList{}, err
	}
	podsToMachines, err := MapPodsToMachines(podLister)

	list := HostPriorityList{}
	for _, node := range nodes.Items {
		list = append(list, calculateOccupancy(pod, node, podsToMachines[node.Name]))
	}
	return list, nil
}

type NodeLabelPrioritizer struct {
	label    string
	presence bool
}

func NewNodeLabelPriority(label string, presence bool) PriorityFunction {
	labelPrioritizer := &NodeLabelPrioritizer{
		label:    label,
		presence: presence,
	}
	return labelPrioritizer.CalculateNodeLabelPriority
}

// CalculateNodeLabelPriority checks whether a particular label exists on a minion or not, regardless of its value.
// If presence is true, prioritizes minions that have the specified label, regardless of value.
// If presence is false, prioritizes minions that do not have the specified label.
func (n *NodeLabelPrioritizer) CalculateNodeLabelPriority(pod api.Pod, podLister PodLister, minionLister MinionLister) (HostPriorityList, error) {
	var score int
	minions, err := minionLister.List()
	if err != nil {
		return nil, err
	}

	labeledMinions := map[string]bool{}
	for _, minion := range minions.Items {
		exists := labels.Set(minion.Labels).Has(n.label)
		labeledMinions[minion.Name] = (exists && n.presence) || (!exists && !n.presence)
	}

	result := []HostPriority{}
	//score int - scale of 0-10
	// 0 being the lowest priority and 10 being the highest
	for minionName, success := range labeledMinions {
		if success {
			score = 10
		} else {
			score = 0
		}
		result = append(result, HostPriority{host: minionName, score: score})
	}
	return result, nil
}

// BalancedResourceAllocation favors nodes with balanced resource usage rate.
// BalancedResourceAllocation should **NOT** be used alone, and **MUST** be used together with LeastRequestedPriority.
// It calculates the difference between the cpu and memory fracion of capacity, and prioritizes the host based on how
// close the two metrics are to each other.
// Detail: score = 10 - abs(cpuFraction-memoryFraction)*10. The algorithm is partly inspired by:
// "Wei Huang et al. An Energy Efficient Virtual Machine Placement Algorithm with Balanced Resource Utilization"
func BalancedResourceAllocation(pod api.Pod, podLister PodLister, minionLister MinionLister) (HostPriorityList, error) {
	nodes, err := minionLister.List()
	if err != nil {
		return HostPriorityList{}, err
	}
	podsToMachines, err := MapPodsToMachines(podLister)

	list := HostPriorityList{}
	for _, node := range nodes.Items {
		list = append(list, calculateBalancedResourceAllocation(pod, node, podsToMachines[node.Name]))
	}
	return list, nil
}

func calculateBalancedResourceAllocation(pod api.Pod, node api.Node, pods []api.Pod) HostPriority {
	totalMilliCPU := int64(0)
	totalMemory := int64(0)
	score := int(0)
	for _, existingPod := range pods {
		for _, container := range existingPod.Spec.Containers {
			totalMilliCPU += container.Resources.Limits.Cpu().MilliValue()
			totalMemory += container.Resources.Limits.Memory().Value()
		}
	}
	// Add the resources requested by the current pod being scheduled.
	// This also helps differentiate between differently sized, but empty, minions.
	for _, container := range pod.Spec.Containers {
		totalMilliCPU += container.Resources.Limits.Cpu().MilliValue()
		totalMemory += container.Resources.Limits.Memory().Value()
	}

	capacityMilliCPU := node.Status.Capacity.Cpu().MilliValue()
	capacityMemory := node.Status.Capacity.Memory().Value()

	cpuFraction := fractionOfCapacity(totalMilliCPU, capacityMilliCPU, node.Name)
	memoryFraction := fractionOfCapacity(totalMemory, capacityMemory, node.Name)
	if cpuFraction >= 1 || memoryFraction >= 1 {
		// if requested >= capacity, the corresponding host should never be preferrred.
		score = 0
	} else {
		// Upper and lower boundary of difference between cpuFraction and memoryFraction are -1 and 1
		// respectively. Multilying the absolute value of the difference by 10 scales the value to
		// 0-10 with 0 representing well balanced allocation and 10 poorly balanced. Subtracting it from
		// 10 leads to the score which also scales from 0 to 10 while 10 representing well balanced.
		diff := math.Abs(cpuFraction - memoryFraction)
		score = int(10 - diff*10)
	}
	glog.V(4).Infof(
		"%v -> %v: Balanced Resource Allocation, Absolute/Requested: (%d, %d) / (%d, %d) Score: (%d)",
		pod.Name, node.Name,
		totalMilliCPU, totalMemory,
		capacityMilliCPU, capacityMemory,
		score,
	)

	return HostPriority{
		host:  node.Name,
		score: score,
	}
}

func fractionOfCapacity(requested, capacity int64, node string) float64 {
	if capacity == 0 {
		return 1
	}
	return float64(requested) / float64(capacity)
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/bitrise-io/bitrise/bitrise"
	models "github.com/bitrise-io/bitrise/models/models_1_0_0"
	envmanModels "github.com/bitrise-io/envman/models"
	"github.com/bitrise-io/go-pathutil/pathutil"
	"github.com/bitrise-io/goinp/goinp"
	stepmanModels "github.com/bitrise-io/stepman/models"
	"github.com/codegangsta/cli"
)

const (
	defaultStepLibSource = "https://bitbucket.org/bitrise-team/bitrise-new-steps-spec"
	//
	defaultSecretsContent = `envs:
- MY_HOME: $HOME
- MY_SECRET_PASSWORD: XyZ
  opts:
    # You can include some options as well if you
    #  want to change how the value is passed to a command.
    is_expand: no
    # For example you can use is_expand: no
    #  if you want to make it sure that
    #  the value is preserved as-it-is, and won't be
    #  expanded before use.
    # For example if your password contains the dollar sign ($)
    #  it would (by default) be expanded as an environment variable,
    #  just like $HOME would be expanded/replaced with your home
    #  directory path.
    # You can prevent this with is_expand: no`
)

func doInit(c *cli.Context) {
	PrintBitriseHeaderASCIIArt()

	bitriseConfigFileRelPath := "./" + DefaultBitriseConfigFileName
	bitriseSecretsFileRelPath := "./" + DefaultSecretsFileName

	if exists, err := pathutil.IsPathExists(bitriseConfigFileRelPath); err != nil {
		log.Fatalln("Error:", err)
	} else if exists {
		ask := fmt.Sprintf("A config file already exists at %s - do you want to overwrite it?", bitriseConfigFileRelPath)
		if val, err := goinp.AskForBool(ask); err != nil {
			log.Fatalln("Error:", err)
		} else if !val {
			log.Infoln("Init canceled, existing file won't be overwritten.")
			os.Exit(0)
		}
	}

	defaultExpand := true
	projectSettingsEnvs := []envmanModels.EnvironmentItemModel{}
	if val, err := goinp.AskForString("What's the BITRISE_PROJECT_TITLE?"); err != nil {
		log.Fatalln(err)
	} else {
		projectTitleEnv := envmanModels.EnvironmentItemModel{
			"BITRISE_PROJECT_TITLE": val,
			"opts": envmanModels.EnvironmentItemOptionsModel{
				IsExpand: &defaultExpand,
			},
		}
		projectSettingsEnvs = append(projectSettingsEnvs, projectTitleEnv)
	}
	if val, err := goinp.AskForString("What's your primary development branch's name?"); err != nil {
		log.Fatalln(err)
	} else {
		devBranchEnv := envmanModels.EnvironmentItemModel{
			"BITRISE_DEV_BRANCH": val,
			"opts": envmanModels.EnvironmentItemOptionsModel{
				IsExpand: &defaultExpand,
			},
		}
		projectSettingsEnvs = append(projectSettingsEnvs, devBranchEnv)
	}

	// TODO:
	//  generate a couple of base steps
	//  * timestamp gen
	//  * bash script - hello world

	scriptStepTitle := "Hello Bitrise!"
	scriptStepContent := `#!/bin/bash
echo "Welcome to Bitrise!"`
	bitriseConf := models.BitriseDataModel{
		FormatVersion:        c.App.Version,
		DefaultStepLibSource: defaultStepLibSource,
		App: models.AppModel{
			Environments: projectSettingsEnvs,
		},
		Workflows: map[string]models.WorkflowModel{
			"primary": models.WorkflowModel{
				Steps: []models.StepListItemModel{
					models.StepListItemModel{
						"script": stepmanModels.StepModel{
							Title: &scriptStepTitle,
							Inputs: []envmanModels.EnvironmentItemModel{
								envmanModels.EnvironmentItemModel{
									"content": scriptStepContent,
								},
							},
						},
					},
				},
			},
		},
	}

	if err := bitrise.SaveConfigToFile(bitriseConfigFileRelPath, bitriseConf); err != nil {
		log.Fatalln("Failed to init the bitrise config file:", err)
	} else {
		fmt.Println()
		fmt.Println("# NOTES about the " + DefaultBitriseConfigFileName + " config file:")
		fmt.Println()
		fmt.Println("We initialized a " + DefaultBitriseConfigFileName + " config file for you.")
		fmt.Println("If you're in this folder you can use this config file")
		fmt.Println(" with bitrise automatically, you don't have to")
		fmt.Println(" specify it's path.")
		fmt.Println()
	}

	if initialized, err := saveSecretsToFile(bitriseSecretsFileRelPath, defaultSecretsContent); err != nil {
		log.Fatalln("Failed to init the secrets file:", err)
	} else if initialized {
		fmt.Println()
		fmt.Println("# NOTES about the " + DefaultSecretsFileName + " secrets file:")
		fmt.Println()
		fmt.Println("We also created a " + DefaultSecretsFileName + " file")
		fmt.Println(" in this directory, to keep your passwords, absolute path configurations")
		fmt.Println(" and other secrets separate from your")
		fmt.Println(" main configuration file.")
		fmt.Println("This way you can safely commit and share your configuration file")
		fmt.Println(" and ignore this secrets file, so nobody else will")
		fmt.Println(" know about your secrets.")
		fmt.Println("You should NEVER commit this secrets file into your repository!!")
		fmt.Println()
	}

	fmt.Println()
	fmt.Println("Hurray, you're good to go!")
	fmt.Println("You can simply run:")
	fmt.Println("-> bitrise run primary")
	fmt.Println("to test the sample configuration (which contains")
	fmt.Println("an example workflow called 'primary').")
	fmt.Println()
	fmt.Println("Once you tested this sample setup you can")
	fmt.Println(" open the " + DefaultBitriseConfigFileName + " config file,")
	fmt.Println(" modify it and then run a workflow with:")
	fmt.Println("-> bitrise run YOUR-WORKFLOW-NAME")
}

func saveSecretsToFile(pth, secretsStr string) (bool, error) {
	if exists, err := pathutil.IsPathExists(pth); err != nil {
		return false, err
	} else if exists {
		ask := fmt.Sprintf("A secrets file already exists at %s - do you want to overwrite it?", pth)
		if val, err := goinp.AskForBool(ask); err != nil {
			return false, err
		} else if !val {
			log.Infoln("Init canceled, existing file (" + pth + ") won't be overwritten.")
			return false, nil
		}
	}

	if err := bitrise.WriteStringToFile(pth, secretsStr); err != nil {
		return false, err
	}
	return true, nil
}
<commit_msg>init highligth<commit_after>package cli

import (
	"fmt"
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/bitrise-io/bitrise/bitrise"
	"github.com/bitrise-io/bitrise/colorstring"
	models "github.com/bitrise-io/bitrise/models/models_1_0_0"
	envmanModels "github.com/bitrise-io/envman/models"
	"github.com/bitrise-io/go-pathutil/pathutil"
	"github.com/bitrise-io/goinp/goinp"
	stepmanModels "github.com/bitrise-io/stepman/models"
	"github.com/codegangsta/cli"
)

const (
	defaultStepLibSource = "https://bitbucket.org/bitrise-team/bitrise-new-steps-spec"
	//
	defaultSecretsContent = `envs:
- MY_HOME: $HOME
- MY_SECRET_PASSWORD: XyZ
  opts:
    # You can include some options as well if you
    #  want to change how the value is passed to a command.
    is_expand: no
    # For example you can use is_expand: no
    #  if you want to make it sure that
    #  the value is preserved as-it-is, and won't be
    #  expanded before use.
    # For example if your password contains the dollar sign ($)
    #  it would (by default) be expanded as an environment variable,
    #  just like $HOME would be expanded/replaced with your home
    #  directory path.
    # You can prevent this with is_expand: no`
)

func doInit(c *cli.Context) {
	PrintBitriseHeaderASCIIArt()

	bitriseConfigFileRelPath := "./" + DefaultBitriseConfigFileName
	bitriseSecretsFileRelPath := "./" + DefaultSecretsFileName

	if exists, err := pathutil.IsPathExists(bitriseConfigFileRelPath); err != nil {
		log.Fatalln("Error:", err)
	} else if exists {
		ask := fmt.Sprintf("A config file already exists at %s - do you want to overwrite it?", bitriseConfigFileRelPath)
		if val, err := goinp.AskForBool(ask); err != nil {
			log.Fatalln("Error:", err)
		} else if !val {
			log.Infoln("Init canceled, existing file won't be overwritten.")
			os.Exit(0)
		}
	}

	defaultExpand := true
	projectSettingsEnvs := []envmanModels.EnvironmentItemModel{}
	if val, err := goinp.AskForString("What's the BITRISE_PROJECT_TITLE?"); err != nil {
		log.Fatalln(err)
	} else {
		projectTitleEnv := envmanModels.EnvironmentItemModel{
			"BITRISE_PROJECT_TITLE": val,
			"opts": envmanModels.EnvironmentItemOptionsModel{
				IsExpand: &defaultExpand,
			},
		}
		projectSettingsEnvs = append(projectSettingsEnvs, projectTitleEnv)
	}
	if val, err := goinp.AskForString("What's your primary development branch's name?"); err != nil {
		log.Fatalln(err)
	} else {
		devBranchEnv := envmanModels.EnvironmentItemModel{
			"BITRISE_DEV_BRANCH": val,
			"opts": envmanModels.EnvironmentItemOptionsModel{
				IsExpand: &defaultExpand,
			},
		}
		projectSettingsEnvs = append(projectSettingsEnvs, devBranchEnv)
	}

	// TODO:
	//  generate a couple of base steps
	//  * timestamp gen
	//  * bash script - hello world

	scriptStepTitle := "Hello Bitrise!"
	scriptStepContent := `#!/bin/bash
echo "Welcome to Bitrise!"`
	bitriseConf := models.BitriseDataModel{
		FormatVersion:        c.App.Version,
		DefaultStepLibSource: defaultStepLibSource,
		App: models.AppModel{
			Environments: projectSettingsEnvs,
		},
		Workflows: map[string]models.WorkflowModel{
			"primary": models.WorkflowModel{
				Steps: []models.StepListItemModel{
					models.StepListItemModel{
						"script": stepmanModels.StepModel{
							Title: &scriptStepTitle,
							Inputs: []envmanModels.EnvironmentItemModel{
								envmanModels.EnvironmentItemModel{
									"content": scriptStepContent,
								},
							},
						},
					},
				},
			},
		},
	}

	if err := bitrise.SaveConfigToFile(bitriseConfigFileRelPath, bitriseConf); err != nil {
		log.Fatalln("Failed to init the bitrise config file:", err)
	} else {
		fmt.Println()
		fmt.Println("# NOTES about the " + DefaultBitriseConfigFileName + " config file:")
		fmt.Println()
		fmt.Println("We initialized a " + DefaultBitriseConfigFileName + " config file for you.")
		fmt.Println("If you're in this folder you can use this config file")
		fmt.Println(" with bitrise automatically, you don't have to")
		fmt.Println(" specify it's path.")
		fmt.Println()
	}

	if initialized, err := saveSecretsToFile(bitriseSecretsFileRelPath, defaultSecretsContent); err != nil {
		log.Fatalln("Failed to init the secrets file:", err)
	} else if initialized {
		fmt.Println()
		fmt.Println("# NOTES about the " + DefaultSecretsFileName + " secrets file:")
		fmt.Println()
		fmt.Println("We also created a " + DefaultSecretsFileName + " file")
		fmt.Println(" in this directory, to keep your passwords, absolute path configurations")
		fmt.Println(" and other secrets separate from your")
		fmt.Println(" main configuration file.")
		fmt.Println("This way you can safely commit and share your configuration file")
		fmt.Println(" and ignore this secrets file, so nobody else will")
		fmt.Println(" know about your secrets.")
		fmt.Println(colorstring.Yellow("You should NEVER commit this secrets file into your repository!!"))
		fmt.Println()
	}

	fmt.Println()
	fmt.Println("Hurray, you're good to go!")
	fmt.Println("You can simply run:")
	fmt.Println("-> bitrise run primary")
	fmt.Println("to test the sample configuration (which contains")
	fmt.Println("an example workflow called 'primary').")
	fmt.Println()
	fmt.Println("Once you tested this sample setup you can")
	fmt.Println(" open the " + DefaultBitriseConfigFileName + " config file,")
	fmt.Println(" modify it and then run a workflow with:")
	fmt.Println("-> bitrise run YOUR-WORKFLOW-NAME")
}

func saveSecretsToFile(pth, secretsStr string) (bool, error) {
	if exists, err := pathutil.IsPathExists(pth); err != nil {
		return false, err
	} else if exists {
		ask := fmt.Sprintf("A secrets file already exists at %s - do you want to overwrite it?", pth)
		if val, err := goinp.AskForBool(ask); err != nil {
			return false, err
		} else if !val {
			log.Infoln("Init canceled, existing file (" + pth + ") won't be overwritten.")
			return false, nil
		}
	}

	if err := bitrise.WriteStringToFile(pth, secretsStr); err != nil {
		return false, err
	}
	return true, nil
}
<|endoftext|>


<commit_before>//
// Package ep is a collection of structures and functions for working with the EPrints REST API
//
// @author R. S. Doiel, <rsdoiel@caltech.edu>
//
// Copyright (c) 2017, Caltech
// All rights not granted herein are expressly reserved by Caltech.
//
// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	// Caltech Library Packages
	"github.com/caltechlibrary/cli"
	ep "github.com/caltechlibrary/eprinttools"
)

var (
	// cli help text
	usage = `USAGE %s [OPTIONS] [EP_EPRINTS_URL|ONE_OR_MORE_EPRINT_ID]`

	description = `
SYNOPSIS

%s wraps the REST API for EPrints 3.3 or better. It can return a list
of uri, a JSON view of the XML presentation as well as generates feeds
and web pages.

CONFIGURATION

ep can be configured with following environment variables

EP_EPRINTS_URL the URL to your EPrints installation

EP_DATASET the dataset and collection name for exporting, site building, and content retrieval`

	examples = `
EXAMPLE

    %s -export all

Would export the entire EPrints repository public content defined by the
environment virables EP_API_URL, EP_DATASET.

    %s -export 2000

Would export 2000 EPrints from the repository with the heighest ID values.

   %s -export-modified 2017-07-01

Would export the EPrint records modified since July 1, 2017.

   %s -export-modified 2017-07-01,2017-07-31 \
      -export-save-keys=july-keys.txt

Would export the EPrint records with modified times in July 2017 and
save the keys for the records exported with one key per line.
`

	// Standard Options
	showHelp     bool
	showVersion  bool
	showLicense  bool
	showExamples bool
	outputFName  string
	verbose      bool

	// App Options
	useAPI      bool
	prettyPrint bool

	apiURL      string
	datasetName string

	updatedSince          string
	exportEPrints         string
	exportEPrintsModified string
	exportSaveKeys        string
	feedSize              int

	authMethod string
	userName   string
	userSecret string
)

func init() {
	// Setup options
	feedSize = ep.DefaultFeedSize

	flag.BoolVar(&showHelp, "h", false, "display help")
	flag.BoolVar(&showHelp, "help", false, "display help")
	flag.BoolVar(&showLicense, "l", false, "display license")
	flag.BoolVar(&showLicense, "license", false, "display license")
	flag.BoolVar(&showVersion, "v", false, "display version")
	flag.BoolVar(&showVersion, "version", false, "display version")
	flag.BoolVar(&showExamples, "example", false, "display example(s)")
	flag.StringVar(&outputFName, "o", "", "output filename (logging)")
	flag.StringVar(&outputFName, "output", "", "output filename (logging)")
	flag.BoolVar(&verbose, "verbose", true, "verbose logging")

	// App Specific options
	flag.StringVar(&authMethod, "auth", "", "set the authentication method (e.g. none, basic, oauth, shib)")
	flag.StringVar(&userName, "username", "", "set the username")
	flag.StringVar(&userName, "un", "", "set the username")
	flag.StringVar(&userSecret, "pw", "", "set the password")

	flag.StringVar(&apiURL, "api", "", "url for EPrints API")
	flag.StringVar(&datasetName, "dataset", "", "dataset/collection name")

	flag.BoolVar(&prettyPrint, "p", false, "pretty print JSON output")
	flag.BoolVar(&prettyPrint, "pretty", false, "pretty print JSON output")
	flag.BoolVar(&useAPI, "read-api", false, "read the contents from the API without saving in the database")
	flag.StringVar(&exportEPrints, "export", "", "export N EPrints from highest ID to lowest")
	flag.StringVar(&exportEPrintsModified, "export-modified", "", "export records by date or date range (e.g. 2017-07-01)")
	flag.StringVar(&exportSaveKeys, "export-save-keys", "", "save the keys exported in a file with provided filename")
	flag.StringVar(&updatedSince, "updated-since", "", "list EPrint IDs updated since a given date (e.g 2017-07-01)")
}

func check(cfg *cli.Config, key, value string) string {
	if value == "" {
		log.Fatalf("Missing %s_%s", cfg.EnvPrefix, strings.ToUpper(key))
		return ""
	}
	return value
}

func main() {
	appName := path.Base(os.Args[0])
	flag.Parse()
	args := flag.Args()

	// Populate cfg from the environment
	cfg := cli.New(appName, "EP", ep.Version)
	cfg.LicenseText = fmt.Sprintf(ep.LicenseText, appName, ep.Version)
	cfg.UsageText = fmt.Sprintf(usage, appName)
	cfg.DescriptionText = fmt.Sprintf(description, appName, appName)
	cfg.OptionText = "OPTIONS"
	cfg.ExampleText = fmt.Sprintf(examples, appName, appName)

	// Handle the default options
	if showHelp == true {
		if len(args) > 0 {
			fmt.Println(cfg.Help(args...))
		} else {
			fmt.Println(cfg.Usage())
		}
		os.Exit(0)
	}

	if showExamples == true {
		if len(args) > 0 {
			fmt.Println(cfg.Example(args...))
		} else {
			fmt.Println(cfg.ExampleText)
		}
		os.Exit(0)
	}

	if showVersion == true {
		fmt.Println(cfg.Version())
		os.Exit(0)
	}
	if showLicense == true {
		fmt.Println(cfg.License())
		os.Exit(0)
	}

	out, err := cli.Create(outputFName, os.Stdout)
	if err != nil {
		fmt.Fprint(os.Stderr, "%s\n", err)
		os.Exit(1)
	}
	defer cli.CloseFile(outputFName, out)

	// Log to out
	log.SetOutput(out)

	// Required configuration
	apiURL = check(cfg, "eprint_url", cfg.MergeEnv("eprint_url", apiURL))
	datasetName = check(cfg, "dataset", cfg.MergeEnv("dataset", datasetName))

	// Optional configuration
	authMethod = cfg.MergeEnv("auth_method", authMethod)
	userName = cfg.MergeEnv("username", userName)
	userSecret = cfg.MergeEnv("password", userSecret)

	// This will read in any settings from the environment
	api, err := ep.New(cfg)
	if err != nil {
		log.Fatal(err)
	}

	if exportEPrints != "" {
		t0 := time.Now()
		exportNo := -1
		if exportEPrints != "all" {
			exportNo, err = strconv.Atoi(exportEPrints)
			if err != nil {
				log.Fatalf("Export count should be %q or an integer, %s", exportEPrints, err)
			}
		}
		log.Printf("%s %s (pid %d)", appName, ep.Version, os.Getpid())
		log.Printf("Export started, %s", t0)
		if err := api.ExportEPrints(exportNo, exportSaveKeys, verbose); err != nil {
			log.Printf("%s", err)
			os.Exit(1)
		}
		log.Printf("Export completed, running time %s", time.Now().Sub(t0))
		os.Exit(0)
	}
	if exportEPrintsModified != "" {
		s := exportEPrintsModified
		e := time.Now().Format("2006-01-02")
		if strings.Contains(s, ",") {
			p := strings.SplitN(s, ",", 2)
			s, e = p[0], p[1]
		}
		start, err := time.Parse("2006-01-02", s)
		if err != nil {
			log.Printf("%s", err)
			os.Exit(1)
		}
		end, err := time.Parse("2006-01-02", e)
		if err != nil {
			log.Printf("%s", err)
			os.Exit(1)
		}
		t0 := time.Now()
		log.Printf("%s %s (pid %d)", appName, ep.Version, os.Getpid())
		log.Printf("Export from %s to %s, started %s", start.Format("2006-01-02"), end.Format("2006-01-02"), t0.Format("2006-01-02 15:04:05 MST"))
		if err := api.ExportModifiedEPrints(start, end, exportSaveKeys, verbose); err != nil {
			log.Printf("%s", err)
			os.Exit(1)
		}
		log.Printf("Export completed, running time %s", time.Now().Sub(t0))
		os.Exit(0)
	}

	//
	// Generate JSON output
	//
	var (
		src  []byte
		data interface{}
	)
	switch {
	case updatedSince != "":
		// date should be formatted YYYY-MM-DD, 2006-01-02
		end := time.Now()
		start, err := time.Parse("2006-01-02", updatedSince)
		if err != nil {
			fmt.Fprintf(os.Stderr, "updated since %q, %s", updatedSince, err)
			os.Exit(1)
		}
		data, err = api.ListModifiedEPrintURI(start, end, verbose)
	case useAPI == true:
		if len(args) == 1 {
			data, _, err = api.GetEPrint(args[0])
		} else {
			data, err = api.ListEPrintsURI()
		}
	default:
		if len(args) == 1 {
			data, err = api.Get(args[0])
		} else if len(args) > 1 {
			records := []*ep.Record{}
			for _, id := range args {
				if rec, err := api.Get(id); err == nil {
					records = append(records, rec)
				} else {
					fmt.Fprintf(os.Stderr, "Can't read EPrint id %s, %s\n", id, err)
				}
			}
			data = records
		} else {
			data, err = api.ListID(0, -1)
		}
	}
	if err != nil {
		log.Fatal(err)
	}

	if prettyPrint == true {
		src, _ = json.MarshalIndent(data, "", "    ")
	} else {
		src, _ = json.Marshal(data)
	}
	fmt.Printf("%s", src)
}
<commit_msg>fixed formatting<commit_after>//
// Package ep is a collection of structures and functions for working with the EPrints REST API
//
// @author R. S. Doiel, <rsdoiel@caltech.edu>
//
// Copyright (c) 2017, Caltech
// All rights not granted herein are expressly reserved by Caltech.
//
// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	// Caltech Library Packages
	"github.com/caltechlibrary/cli"
	ep "github.com/caltechlibrary/eprinttools"
)

var (
	// cli help text
	usage = `USAGE %s [OPTIONS] [EP_EPRINTS_URL|ONE_OR_MORE_EPRINT_ID]`

	description = `
SYNOPSIS

%s wraps the REST API for EPrints 3.3 or better. It can return a list
of uri, a JSON view of the XML presentation as well as generates feeds
and web pages.

CONFIGURATION

ep can be configured with following environment variables

EP_EPRINTS_URL the URL to your EPrints installation

EP_DATASET the dataset and collection name for exporting, site building, and content retrieval`

	examples = `
EXAMPLE

    %s -export all

Would export the entire EPrints repository public content defined by the
environment virables EP_API_URL, EP_DATASET.

    %s -export 2000

Would export 2000 EPrints from the repository with the heighest ID values.

   %s -export-modified 2017-07-01

Would export the EPrint records modified since July 1, 2017.

   %s -export-modified 2017-07-01,2017-07-31 \
      -export-save-keys=july-keys.txt

Would export the EPrint records with modified times in July 2017 and
save the keys for the records exported with one key per line.
`

	// Standard Options
	showHelp     bool
	showVersion  bool
	showLicense  bool
	showExamples bool
	outputFName  string
	verbose      bool

	// App Options
	useAPI      bool
	prettyPrint bool

	apiURL      string
	datasetName string

	updatedSince          string
	exportEPrints         string
	exportEPrintsModified string
	exportSaveKeys        string
	feedSize              int

	authMethod string
	userName   string
	userSecret string
)

func init() {
	// Setup options
	feedSize = ep.DefaultFeedSize

	flag.BoolVar(&showHelp, "h", false, "display help")
	flag.BoolVar(&showHelp, "help", false, "display help")
	flag.BoolVar(&showLicense, "l", false, "display license")
	flag.BoolVar(&showLicense, "license", false, "display license")
	flag.BoolVar(&showVersion, "v", false, "display version")
	flag.BoolVar(&showVersion, "version", false, "display version")
	flag.BoolVar(&showExamples, "example", false, "display example(s)")
	flag.StringVar(&outputFName, "o", "", "output filename (logging)")
	flag.StringVar(&outputFName, "output", "", "output filename (logging)")
	flag.BoolVar(&verbose, "verbose", true, "verbose logging")

	// App Specific options
	flag.StringVar(&authMethod, "auth", "", "set the authentication method (e.g. none, basic, oauth, shib)")
	flag.StringVar(&userName, "username", "", "set the username")
	flag.StringVar(&userName, "un", "", "set the username")
	flag.StringVar(&userSecret, "pw", "", "set the password")

	flag.StringVar(&apiURL, "api", "", "url for EPrints API")
	flag.StringVar(&datasetName, "dataset", "", "dataset/collection name")

	flag.BoolVar(&prettyPrint, "p", false, "pretty print JSON output")
	flag.BoolVar(&prettyPrint, "pretty", false, "pretty print JSON output")
	flag.BoolVar(&useAPI, "read-api", false, "read the contents from the API without saving in the database")
	flag.StringVar(&exportEPrints, "export", "", "export N EPrints from highest ID to lowest")
	flag.StringVar(&exportEPrintsModified, "export-modified", "", "export records by date or date range (e.g. 2017-07-01)")
	flag.StringVar(&exportSaveKeys, "export-save-keys", "", "save the keys exported in a file with provided filename")
	flag.StringVar(&updatedSince, "updated-since", "", "list EPrint IDs updated since a given date (e.g 2017-07-01)")
}

func check(cfg *cli.Config, key, value string) string {
	if value == "" {
		log.Fatalf("Missing %s_%s", cfg.EnvPrefix, strings.ToUpper(key))
		return ""
	}
	return value
}

func main() {
	appName := path.Base(os.Args[0])
	flag.Parse()
	args := flag.Args()

	// Populate cfg from the environment
	cfg := cli.New(appName, "EP", ep.Version)
	cfg.LicenseText = fmt.Sprintf(ep.LicenseText, appName, ep.Version)
	cfg.UsageText = fmt.Sprintf(usage, appName)
	cfg.DescriptionText = fmt.Sprintf(description, appName, appName)
	cfg.OptionText = "OPTIONS"
	cfg.ExampleText = fmt.Sprintf(examples, appName, appName, appName, appName)

	// Handle the default options
	if showHelp == true {
		if len(args) > 0 {
			fmt.Println(cfg.Help(args...))
		} else {
			fmt.Println(cfg.Usage())
		}
		os.Exit(0)
	}

	if showExamples == true {
		if len(args) > 0 {
			fmt.Println(cfg.Example(args...))
		} else {
			fmt.Println(cfg.ExampleText)
		}
		os.Exit(0)
	}

	if showVersion == true {
		fmt.Println(cfg.Version())
		os.Exit(0)
	}
	if showLicense == true {
		fmt.Println(cfg.License())
		os.Exit(0)
	}

	out, err := cli.Create(outputFName, os.Stdout)
	if err != nil {
		fmt.Fprint(os.Stderr, "%s\n", err)
		os.Exit(1)
	}
	defer cli.CloseFile(outputFName, out)

	// Log to out
	log.SetOutput(out)

	// Required configuration
	apiURL = check(cfg, "eprint_url", cfg.MergeEnv("eprint_url", apiURL))
	datasetName = check(cfg, "dataset", cfg.MergeEnv("dataset", datasetName))

	// Optional configuration
	authMethod = cfg.MergeEnv("auth_method", authMethod)
	userName = cfg.MergeEnv("username", userName)
	userSecret = cfg.MergeEnv("password", userSecret)

	// This will read in any settings from the environment
	api, err := ep.New(cfg)
	if err != nil {
		log.Fatal(err)
	}

	if exportEPrints != "" {
		t0 := time.Now()
		exportNo := -1
		if exportEPrints != "all" {
			exportNo, err = strconv.Atoi(exportEPrints)
			if err != nil {
				log.Fatalf("Export count should be %q or an integer, %s", exportEPrints, err)
			}
		}
		log.Printf("%s %s (pid %d)", appName, ep.Version, os.Getpid())
		log.Printf("Export started, %s", t0)
		if err := api.ExportEPrints(exportNo, exportSaveKeys, verbose); err != nil {
			log.Printf("%s", err)
			os.Exit(1)
		}
		log.Printf("Export completed, running time %s", time.Now().Sub(t0))
		os.Exit(0)
	}
	if exportEPrintsModified != "" {
		s := exportEPrintsModified
		e := time.Now().Format("2006-01-02")
		if strings.Contains(s, ",") {
			p := strings.SplitN(s, ",", 2)
			s, e = p[0], p[1]
		}
		start, err := time.Parse("2006-01-02", s)
		if err != nil {
			log.Printf("%s", err)
			os.Exit(1)
		}
		end, err := time.Parse("2006-01-02", e)
		if err != nil {
			log.Printf("%s", err)
			os.Exit(1)
		}
		t0 := time.Now()
		log.Printf("%s %s (pid %d)", appName, ep.Version, os.Getpid())
		log.Printf("Export from %s to %s, started %s", start.Format("2006-01-02"), end.Format("2006-01-02"), t0.Format("2006-01-02 15:04:05 MST"))
		if err := api.ExportModifiedEPrints(start, end, exportSaveKeys, verbose); err != nil {
			log.Printf("%s", err)
			os.Exit(1)
		}
		log.Printf("Export completed, running time %s", time.Now().Sub(t0))
		os.Exit(0)
	}

	//
	// Generate JSON output
	//
	var (
		src  []byte
		data interface{}
	)
	switch {
	case updatedSince != "":
		// date should be formatted YYYY-MM-DD, 2006-01-02
		end := time.Now()
		start, err := time.Parse("2006-01-02", updatedSince)
		if err != nil {
			fmt.Fprintf(os.Stderr, "updated since %q, %s", updatedSince, err)
			os.Exit(1)
		}
		data, err = api.ListModifiedEPrintURI(start, end, verbose)
	case useAPI == true:
		if len(args) == 1 {
			data, _, err = api.GetEPrint(args[0])
		} else {
			data, err = api.ListEPrintsURI()
		}
	default:
		if len(args) == 1 {
			data, err = api.Get(args[0])
		} else if len(args) > 1 {
			records := []*ep.Record{}
			for _, id := range args {
				if rec, err := api.Get(id); err == nil {
					records = append(records, rec)
				} else {
					fmt.Fprintf(os.Stderr, "Can't read EPrint id %s, %s\n", id, err)
				}
			}
			data = records
		} else {
			data, err = api.ListID(0, -1)
		}
	}
	if err != nil {
		log.Fatal(err)
	}

	if prettyPrint == true {
		src, _ = json.MarshalIndent(data, "", "    ")
	} else {
		src, _ = json.Marshal(data)
	}
	fmt.Printf("%s", src)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package matcher

import (
	"fmt"
	"time"

	"github.com/m3db/m3cluster/client"
	"github.com/m3db/m3cluster/kv"
	"github.com/m3db/m3metrics/filters"
	"github.com/m3db/m3metrics/metric/id"
	"github.com/m3db/m3metrics/metric/id/m3"
	"github.com/m3db/m3metrics/policy"
	"github.com/m3db/m3metrics/rules"
	"github.com/m3db/m3x/clock"
	"github.com/m3db/m3x/instrument"
	"github.com/m3db/m3x/pool"
)

// Configuration is config used to create a Matcher.
type Configuration struct {
	InitWatchTimeout      time.Duration                        `yaml:"initWatchTimeout"`
	RulesKVConfig         kv.Configuration                     `yaml:"rulesKVConfig"`
	NamespacesKey         string                               `yaml:"namespacesKey" validate:"nonzero"`
	RuleSetKeyFmt         string                               `yaml:"ruleSetKeyFmt" validate:"nonzero"`
	NamespaceTag          string                               `yaml:"namespaceTag" validate:"nonzero"`
	DefaultNamespace      string                               `yaml:"defaultNamespace" validate:"nonzero"`
	NameTagKey            string                               `yaml:"nameTagKey" validate:"nonzero"`
	MatchRangePast        *time.Duration                       `yaml:"matchRangePast"`
	SortedTagIteratorPool pool.ObjectPoolConfiguration         `yaml:"sortedTagIteratorPool"`
	AggregationTypes      policy.AggregationTypesConfiguration `yaml:"aggregationTypes"`
}

// NewNamespaces creates a matcher.Namespaces.
func (cfg *Configuration) NewNamespaces(
	kvCluster client.Client,
	clockOpts clock.Options,
	instrumentOpts instrument.Options,
) (Namespaces, error) {
	opts, err := cfg.NewOptions(kvCluster, clockOpts, instrumentOpts)
	if err != nil {
		return nil, err
	}

	namespaces := NewNamespaces(opts.NamespacesKey(), opts)
	return namespaces, nil
}

// NewMatcher creates a Matcher.
func (cfg *Configuration) NewMatcher(
	cache Cache,
	kvCluster client.Client,
	clockOpts clock.Options,
	instrumentOpts instrument.Options,
) (Matcher, error) {
	opts, err := cfg.NewOptions(kvCluster, clockOpts, instrumentOpts)
	if err != nil {
		return nil, err
	}

	return NewMatcher(cache, opts)
}

// NewOptions creates a Options.
func (cfg *Configuration) NewOptions(
	kvCluster client.Client,
	clockOpts clock.Options,
	instrumentOpts instrument.Options,
) (Options, error) {
	// Configure rules kv store.
	kvOpts, err := cfg.RulesKVConfig.NewOptions()
	if err != nil {
		return nil, err
	}

	if err := kvOpts.Validate(); err != nil {
		return nil, err
	}

	rulesStore, err := kvCluster.Store(kvOpts)
	if err != nil {
		return nil, err
	}

	// Configure rules options.
	scope := instrumentOpts.MetricsScope().SubScope("sorted-tag-iterator-pool")
	poolOpts := cfg.SortedTagIteratorPool.NewObjectPoolOptions(instrumentOpts.SetMetricsScope(scope))
	sortedTagIteratorPool := id.NewSortedTagIteratorPool(poolOpts)
	sortedTagIteratorPool.Init(func() id.SortedTagIterator {
		return m3.NewPooledSortedTagIterator(nil, sortedTagIteratorPool)
	})
	sortedTagIteratorFn := func(tagPairs []byte) id.SortedTagIterator {
		it := sortedTagIteratorPool.Get()
		it.Reset(tagPairs)
		return it
	}
	tagsFilterOptions := filters.TagsFilterOptions{
		NameTagKey:          []byte(cfg.NameTagKey),
		NameAndTagsFn:       m3.NameAndTags,
		SortedTagIteratorFn: sortedTagIteratorFn,
	}

	isRollupIDFn := func(name []byte, tags []byte) bool {
		return m3.IsRollupID(name, tags, sortedTagIteratorPool)
	}

	aggTypeOpts, err := cfg.AggregationTypes.NewOptions(instrumentOpts)
	if err != nil {
		return nil, err
	}
	ruleSetOpts := rules.NewOptions().
		SetTagsFilterOptions(tagsFilterOptions).
		SetNewRollupIDFn(m3.NewRollupID).
		SetIsRollupIDFn(isRollupIDFn).
		SetAggregationTypesOptions(aggTypeOpts)

	// Configure ruleset key function.
	ruleSetKeyFn := func(namespace []byte) string {
		return fmt.Sprintf(cfg.RuleSetKeyFmt, namespace)
	}

	opts := NewOptions().
		SetClockOptions(clockOpts).
		SetInstrumentOptions(instrumentOpts).
		SetRuleSetOptions(ruleSetOpts).
		SetKVStore(rulesStore).
		SetNamespacesKey(cfg.NamespacesKey).
		SetRuleSetKeyFn(ruleSetKeyFn).
		SetNamespaceTag([]byte(cfg.NamespaceTag)).
		SetDefaultNamespace([]byte(cfg.DefaultNamespace))

	if cfg.InitWatchTimeout != 0 {
		opts = opts.SetInitWatchTimeout(cfg.InitWatchTimeout)
	}
	if cfg.MatchRangePast != nil {
		opts = opts.SetMatchRangePast(*cfg.MatchRangePast)
	}

	return opts, nil
}
<commit_msg>Remove options validation since it's done inside m3cluster (#118)<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package matcher

import (
	"fmt"
	"time"

	"github.com/m3db/m3cluster/client"
	"github.com/m3db/m3cluster/kv"
	"github.com/m3db/m3metrics/filters"
	"github.com/m3db/m3metrics/metric/id"
	"github.com/m3db/m3metrics/metric/id/m3"
	"github.com/m3db/m3metrics/policy"
	"github.com/m3db/m3metrics/rules"
	"github.com/m3db/m3x/clock"
	"github.com/m3db/m3x/instrument"
	"github.com/m3db/m3x/pool"
)

// Configuration is config used to create a Matcher.
type Configuration struct {
	InitWatchTimeout      time.Duration                        `yaml:"initWatchTimeout"`
	RulesKVConfig         kv.Configuration                     `yaml:"rulesKVConfig"`
	NamespacesKey         string                               `yaml:"namespacesKey" validate:"nonzero"`
	RuleSetKeyFmt         string                               `yaml:"ruleSetKeyFmt" validate:"nonzero"`
	NamespaceTag          string                               `yaml:"namespaceTag" validate:"nonzero"`
	DefaultNamespace      string                               `yaml:"defaultNamespace" validate:"nonzero"`
	NameTagKey            string                               `yaml:"nameTagKey" validate:"nonzero"`
	MatchRangePast        *time.Duration                       `yaml:"matchRangePast"`
	SortedTagIteratorPool pool.ObjectPoolConfiguration         `yaml:"sortedTagIteratorPool"`
	AggregationTypes      policy.AggregationTypesConfiguration `yaml:"aggregationTypes"`
}

// NewNamespaces creates a matcher.Namespaces.
func (cfg *Configuration) NewNamespaces(
	kvCluster client.Client,
	clockOpts clock.Options,
	instrumentOpts instrument.Options,
) (Namespaces, error) {
	opts, err := cfg.NewOptions(kvCluster, clockOpts, instrumentOpts)
	if err != nil {
		return nil, err
	}

	namespaces := NewNamespaces(opts.NamespacesKey(), opts)
	return namespaces, nil
}

// NewMatcher creates a Matcher.
func (cfg *Configuration) NewMatcher(
	cache Cache,
	kvCluster client.Client,
	clockOpts clock.Options,
	instrumentOpts instrument.Options,
) (Matcher, error) {
	opts, err := cfg.NewOptions(kvCluster, clockOpts, instrumentOpts)
	if err != nil {
		return nil, err
	}

	return NewMatcher(cache, opts)
}

// NewOptions creates a Options.
func (cfg *Configuration) NewOptions(
	kvCluster client.Client,
	clockOpts clock.Options,
	instrumentOpts instrument.Options,
) (Options, error) {
	// Configure rules kv store.
	kvOpts, err := cfg.RulesKVConfig.NewOptions()
	if err != nil {
		return nil, err
	}

	rulesStore, err := kvCluster.Store(kvOpts)
	if err != nil {
		return nil, err
	}

	// Configure rules options.
	scope := instrumentOpts.MetricsScope().SubScope("sorted-tag-iterator-pool")
	poolOpts := cfg.SortedTagIteratorPool.NewObjectPoolOptions(instrumentOpts.SetMetricsScope(scope))
	sortedTagIteratorPool := id.NewSortedTagIteratorPool(poolOpts)
	sortedTagIteratorPool.Init(func() id.SortedTagIterator {
		return m3.NewPooledSortedTagIterator(nil, sortedTagIteratorPool)
	})
	sortedTagIteratorFn := func(tagPairs []byte) id.SortedTagIterator {
		it := sortedTagIteratorPool.Get()
		it.Reset(tagPairs)
		return it
	}
	tagsFilterOptions := filters.TagsFilterOptions{
		NameTagKey:          []byte(cfg.NameTagKey),
		NameAndTagsFn:       m3.NameAndTags,
		SortedTagIteratorFn: sortedTagIteratorFn,
	}

	isRollupIDFn := func(name []byte, tags []byte) bool {
		return m3.IsRollupID(name, tags, sortedTagIteratorPool)
	}

	aggTypeOpts, err := cfg.AggregationTypes.NewOptions(instrumentOpts)
	if err != nil {
		return nil, err
	}
	ruleSetOpts := rules.NewOptions().
		SetTagsFilterOptions(tagsFilterOptions).
		SetNewRollupIDFn(m3.NewRollupID).
		SetIsRollupIDFn(isRollupIDFn).
		SetAggregationTypesOptions(aggTypeOpts)

	// Configure ruleset key function.
	ruleSetKeyFn := func(namespace []byte) string {
		return fmt.Sprintf(cfg.RuleSetKeyFmt, namespace)
	}

	opts := NewOptions().
		SetClockOptions(clockOpts).
		SetInstrumentOptions(instrumentOpts).
		SetRuleSetOptions(ruleSetOpts).
		SetKVStore(rulesStore).
		SetNamespacesKey(cfg.NamespacesKey).
		SetRuleSetKeyFn(ruleSetKeyFn).
		SetNamespaceTag([]byte(cfg.NamespaceTag)).
		SetDefaultNamespace([]byte(cfg.DefaultNamespace))

	if cfg.InitWatchTimeout != 0 {
		opts = opts.SetInitWatchTimeout(cfg.InitWatchTimeout)
	}
	if cfg.MatchRangePast != nil {
		opts = opts.SetMatchRangePast(*cfg.MatchRangePast)
	}

	return opts, nil
}
<|endoftext|>