<commit_before>// Copyright 2012-2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"gopkg.in/mgo.v2"

	"github.com/juju/juju/state/bakerystorage"
)

// The capped collection used for transaction logs defaults to 10MB.
// It's tweaked in export_test.go to 1MB to avoid the overhead of
// creating and deleting the large file repeatedly in tests.
var (
	txnLogSize      = 10000000
	txnLogSizeTests = 1000000
)

// allCollections should be the single source of truth for information about
// any collection we use. It's broken up into 4 main sections:
//
//  * infrastructure: we really don't have any business touching these once
//    we've created them. They should have the rawAccess attribute set, so that
//    multiModelRunner will consider them forbidden.
//
//  * global: these hold information external to models. They may include
//    model metadata, or references; but they're generally not relevant
//    from the perspective of a given model.
//
//  * local (in opposition to global; and for want of a better term): these
//    hold information relevant *within* specific models (machines,
//    services, relations, settings, bookkeeping, etc) and should generally be
//    read via an modelStateCollection, and written via a multiModelRunner. This is
//    the most common form of collection, and the above access should usually
//    be automatic via Database.Collection and Database.Runner.
//
//  * raw-access: there's certainly data that's a poor fit for mgo/txn. Most
//    forms of logs, for example, will benefit both from the speedy insert and
//    worry-free bulk deletion; so raw-access collections are fine. Just don't
//    try to run transactions that reference them.
//
// Please do not use collections not referenced here; and when adding new
// collections, please document them, and make an effort to put them in an
// appropriate section.
func allCollections() collectionSchema {
	return collectionSchema{

		// Infrastructure collections
		// ==========================

		txnsC: {
			// This collection is used exclusively by mgo/txn to record transactions.
			global:         true,
			rawAccess:      true,
			explicitCreate: &mgo.CollectionInfo{},
		},
		txnLogC: {
			// This collection is used by mgo/txn to record the set of documents
			// affected by each successful transaction; and by state/watcher to
			// generate a stream of document-resolution events that are delivered
			// to, and interpreted by, both state and state/multiwatcher.
			global:    true,
			rawAccess: true,
			explicitCreate: &mgo.CollectionInfo{
				Capped:   true,
				MaxBytes: txnLogSize,
			},
		},

		// ------------------

		// Global collections
		// ==================

		// This collection holds the details of the controllers hosting, well,
		// everything in state.
		controllersC: {global: true},

		// This collection is used to track progress when restoring a
		// controller from backup.
		restoreInfoC: {global: true},

		// This collection is used by the controllers to coordinate binary
		// upgrades and schema migrations.
		upgradeInfoC: {global: true},

		// This collection holds a convenient representation of the content of
		// the simplestreams data source pointing to binaries required by juju.
		//
		// Tools metadata is per-model, to allow multiple revisions of tools to
		// be uploaded to different models without affecting other models.
		toolsmetadataC: {},

		// This collection holds a convenient representation of the content of
		// the simplestreams data source pointing to Juju GUI archives.
		guimetadataC: {global: true},

		// This collection holds Juju GUI current version and other settings.
		guisettingsC: {global: true},

		// This collection holds model information; in particular its
		// Life and its UUID.
		modelsC: {global: true},

		// This collection holds references to entities owned by a
		// model. We use this to determine whether or not we can safely
		// destroy empty models.
		modelEntityRefsC: {global: true},

		// This collection is holds the parameters for model migrations.
		migrationsC: {
			global: true,
			indexes: []mgo.Index{{
				Key: []string{"model-uuid"},
			}},
		},

		// This collection tracks the progress of model migrations.
		migrationsStatusC: {global: true},

		// This collection records the model migrations which
		// are currently in progress. It is used to ensure that only
		// one model migration document exists per environment.
		migrationsActiveC: {global: true},

		// This collection holds user information that's not specific to any
		// one model.
		usersC: {
			global: true,
			indexes: []mgo.Index{{
				// TODO(thumper): schema change to remove this index.
				Key: []string{"name"},
			}},
		},

		// This collection holds the last time the user connected to the API server.
		userLastLoginC: {
			global:    true,
			rawAccess: true,
		},

		// This collection is used as a unique key restraint. The _id field is
		// a concatenation of multiple fields that form a compound index,
		// allowing us to ensure users cannot have the same name for two
		// different models at a time.
		usermodelnameC: {global: true},

		// This collection holds workload metrics reported by certain charms
		// for passing onward to other tools.
		metricsC: {global: true},

		// This collection holds persistent state for the metrics manager.
		metricsManagerC: {global: true},

		// This collection holds lease data, which is per-model, but is
		// not itself multi-model-aware; happily it will imminently be
		// deprecated in favour of the non-global leasesC below.
		// TODO(fwereade): drop leaseC entirely so can't use wrong const.
		leaseC: {global: true},

		// This collection was deprecated before multi-model support
		// was implemented.
		actionresultsC: {global: true},

		// This collection holds storage items for a macaroon bakery.
		bakeryStorageItemsC: {
			global:  true,
			indexes: bakerystorage.MongoIndexes(),
		},

		// -----------------

		// Local collections
		// =================

		// This collection is basically a standard SQL intersection table; it
		// references the global records of the users allowed access to a
		// given collection.
		modelUsersC: {},

		// This collection holds the last time the model user connected
		// to the model.
		modelUserLastConnectionC: {
			rawAccess: true,
		},

		// This collection contains governors that prevent certain kinds of
		// changes from being accepted.
		blocksC: {},

		// This collection is used for internal bookkeeping; certain complex
		// or tedious state changes are deferred by recording a cleanup doc
		// for later handling.
		cleanupsC: {},

		// This collection contains incrementing integers, subdivided by name,
		// to ensure various IDs aren't reused.
		sequenceC: {},

		// This collection holds lease data. It's currently only used to
		// implement service leadership, but is namespaced and available
		// for use by other clients in future.
		leasesC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "type"},
			}, {
				Key: []string{"model-uuid", "namespace"},
			}},
		},

		// -----

		// These collections hold information associated with services.
		charmsC:   {},
		servicesC: {},
		unitsC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "service"},
			}, {
				Key: []string{"model-uuid", "principal"},
			}, {
				Key: []string{"model-uuid", "machineid"},
			}},
		},
		minUnitsC: {},

		// This collection holds documents that indicate units which are queued
		// to be assigned to machines. It is used exclusively by the
		// AssignUnitWorker.
		assignUnitC: {},

		// meterStatusC is the collection used to store meter status information.
		meterStatusC:  {},
		settingsrefsC: {},
		relationsC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "endpoints.relationname"},
			}, {
				Key: []string{"model-uuid", "endpoints.servicename"},
			}},
		},
		relationScopesC: {},

		// -----

		// These collections hold information associated with machines.
		containerRefsC: {},
		instanceDataC:  {},
		machinesC:      {},
		rebootC:        {},
		sshHostKeysC:   {},

		// -----

		// These collections hold information associated with storage.
		blockDevicesC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "machineid"},
			}},
		},
		filesystemsC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "storageid"},
			}},
		},
		filesystemAttachmentsC: {},
		storageInstancesC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "owner"},
			}},
		},
		storageAttachmentsC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "storageid"},
			}, {
				Key: []string{"model-uuid", "unitid"},
			}},
		},
		volumesC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "storageid"},
			}},
		},
		volumeAttachmentsC: {},

		// -----

		// These collections hold information associated with networking.
		// TODO(dimitern): Remove the obsolete collections below once possible.
		legacyipaddressesC: {
			indexes: []mgo.Index{{
				Key: []string{"uuid"},
			}, {
				Key: []string{"model-uuid", "state"},
			}, {
				Key: []string{"model-uuid", "subnetid"},
			}},
		},
		// TODO(dimitern): End of obsolete networking collections.
		providerIDsC: {
			indexes: []mgo.Index{{
				Key:    []string{"providerid"},
				Unique: true,
			}},
		},
		spacesC: {},
		subnetsC: {
			indexes: []mgo.Index{{
				Key:    []string{"providerid"},
				Unique: true,
				Sparse: true,
			}},
		},
		linkLayerDevicesC: {
			indexes: []mgo.Index{{
				Key:    []string{"providerid"},
				Unique: true,
				Sparse: true,
			}},
		},
		linkLayerDevicesRefsC: {},
		ipAddressesC: {
			indexes: []mgo.Index{{
				Key:    []string{"providerid"},
				Unique: true,
				Sparse: true,
			}},
		},
		endpointBindingsC: {},
		openedPortsC:      {},

		// -----

		// These collections hold information associated with actions.
		actionsC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "name"},
			}},
		},
		actionNotificationsC: {},

		// -----

		// TODO(ericsnow) Use a component-oriented registration mechanism...

		// This collection holds information associated with charm payloads.
		// See payload/persistence/mongo.go.
		"payloads": {},

		// This collection holds information associated with charm resources.
		// See resource/persistence/mongo.go.
		"resources": {},

		// -----

		// The remaining non-global collections share the property of being
		// relevant to multiple other kinds of entities, and are thus generally
		// indexed by globalKey(). This is unhelpfully named in this context --
		// it's meant to imply "global within an model", because it was
		// named before multi-env support.

		// This collection holds user annotations for various entities. They
		// shouldn't be written or interpreted by juju.
		annotationsC: {},

		// This collection in particular holds an astounding number of
		// different sorts of data: service config settings by charm version,
		// unit relation settings, model config, etc etc etc.
		settingsC: {},

		constraintsC:        {},
		storageConstraintsC: {},
		statusesC:           {},
		statusesHistoryC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "globalkey"},
			}},
		},

		// This collection holds information about cloud image metadata.
		cloudimagemetadataC: {},

		// ----------------------

		// Raw-access collections
		// ======================

		// metrics; status-history; logs; ..?
	}
}

// These constants are used to avoid sprinkling the package with any more
// magic strings. If a collection deserves documentation, please document
// it in allCollections, above; and please keep this list sorted for easy
// inspection.
const (
	actionNotificationsC     = "actionnotifications"
	actionresultsC           = "actionresults"
	actionsC                 = "actions"
	annotationsC             = "annotations"
	assignUnitC              = "assignUnits"
	bakeryStorageItemsC      = "bakeryStorageItems"
	blockDevicesC            = "blockdevices"
	blocksC                  = "blocks"
	charmsC                  = "charms"
	cleanupsC                = "cleanups"
	cloudimagemetadataC      = "cloudimagemetadata"
	constraintsC             = "constraints"
	containerRefsC           = "containerRefs"
	controllersC             = "controllers"
	filesystemAttachmentsC   = "filesystemAttachments"
	filesystemsC             = "filesystems"
	guimetadataC             = "guimetadata"
	guisettingsC             = "guisettings"
	instanceDataC            = "instanceData"
	legacyipaddressesC       = "ipaddresses"
	leaseC                   = "lease"
	leasesC                  = "leases"
	machinesC                = "machines"
	meterStatusC             = "meterStatus"
	metricsC                 = "metrics"
	metricsManagerC          = "metricsmanager"
	minUnitsC                = "minunits"
	migrationsStatusC        = "migrations.status"
	migrationsActiveC        = "migrations.active"
	migrationsC              = "migrations"
	modelUserLastConnectionC = "modelUserLastConnection"
	modelUsersC              = "modelusers"
	modelsC                  = "models"
	modelEntityRefsC         = "modelEntityRefs"
	openedPortsC             = "openedPorts"
	providerIDsC             = "providerIDs"
	rebootC                  = "reboot"
	relationScopesC          = "relationscopes"
	relationsC               = "relations"
	restoreInfoC             = "restoreInfo"
	sequenceC                = "sequence"
	servicesC                = "services"
	endpointBindingsC        = "endpointbindings"
	settingsC                = "settings"
	settingsrefsC            = "settingsrefs"
	sshHostKeysC             = "sshhostkeys"
	spacesC                  = "spaces"
	statusesC                = "statuses"
	statusesHistoryC         = "statuseshistory"
	storageAttachmentsC      = "storageattachments"
	storageConstraintsC      = "storageconstraints"
	storageInstancesC        = "storageinstances"
	subnetsC                 = "subnets"
	linkLayerDevicesC        = "linklayerdevices"
	linkLayerDevicesRefsC    = "linklayerdevicesrefs"
	ipAddressesC             = "ip.addresses"
	toolsmetadataC           = "toolsmetadata"
	txnLogC                  = "txns.log"
	txnsC                    = "txns"
	unitsC                   = "units"
	upgradeInfoC             = "upgradeInfo"
	userLastLoginC           = "userLastLogin"
	usermodelnameC           = "usermodelname"
	usersC                   = "users"
	volumeAttachmentsC       = "volumeattachments"
	volumesC                 = "volumes"
	// "payloads" (see payload/persistence/mongo.go)
	// "resources" (see resource/persistence/mongo.go)
)
<commit_msg>Remove index<commit_after>// Copyright 2012-2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package state

import (
	"gopkg.in/mgo.v2"

	"github.com/juju/juju/state/bakerystorage"
)

// The capped collection used for transaction logs defaults to 10MB.
// It's tweaked in export_test.go to 1MB to avoid the overhead of
// creating and deleting the large file repeatedly in tests.
var (
	txnLogSize      = 10000000
	txnLogSizeTests = 1000000
)

// allCollections should be the single source of truth for information about
// any collection we use. It's broken up into 4 main sections:
//
//  * infrastructure: we really don't have any business touching these once
//    we've created them. They should have the rawAccess attribute set, so that
//    multiModelRunner will consider them forbidden.
//
//  * global: these hold information external to models. They may include
//    model metadata, or references; but they're generally not relevant
//    from the perspective of a given model.
//
//  * local (in opposition to global; and for want of a better term): these
//    hold information relevant *within* specific models (machines,
//    services, relations, settings, bookkeeping, etc) and should generally be
//    read via an modelStateCollection, and written via a multiModelRunner. This is
//    the most common form of collection, and the above access should usually
//    be automatic via Database.Collection and Database.Runner.
//
//  * raw-access: there's certainly data that's a poor fit for mgo/txn. Most
//    forms of logs, for example, will benefit both from the speedy insert and
//    worry-free bulk deletion; so raw-access collections are fine. Just don't
//    try to run transactions that reference them.
//
// Please do not use collections not referenced here; and when adding new
// collections, please document them, and make an effort to put them in an
// appropriate section.
func allCollections() collectionSchema {
	return collectionSchema{

		// Infrastructure collections
		// ==========================

		txnsC: {
			// This collection is used exclusively by mgo/txn to record transactions.
			global:         true,
			rawAccess:      true,
			explicitCreate: &mgo.CollectionInfo{},
		},
		txnLogC: {
			// This collection is used by mgo/txn to record the set of documents
			// affected by each successful transaction; and by state/watcher to
			// generate a stream of document-resolution events that are delivered
			// to, and interpreted by, both state and state/multiwatcher.
			global:    true,
			rawAccess: true,
			explicitCreate: &mgo.CollectionInfo{
				Capped:   true,
				MaxBytes: txnLogSize,
			},
		},

		// ------------------

		// Global collections
		// ==================

		// This collection holds the details of the controllers hosting, well,
		// everything in state.
		controllersC: {global: true},

		// This collection is used to track progress when restoring a
		// controller from backup.
		restoreInfoC: {global: true},

		// This collection is used by the controllers to coordinate binary
		// upgrades and schema migrations.
		upgradeInfoC: {global: true},

		// This collection holds a convenient representation of the content of
		// the simplestreams data source pointing to binaries required by juju.
		//
		// Tools metadata is per-model, to allow multiple revisions of tools to
		// be uploaded to different models without affecting other models.
		toolsmetadataC: {},

		// This collection holds a convenient representation of the content of
		// the simplestreams data source pointing to Juju GUI archives.
		guimetadataC: {global: true},

		// This collection holds Juju GUI current version and other settings.
		guisettingsC: {global: true},

		// This collection holds model information; in particular its
		// Life and its UUID.
		modelsC: {global: true},

		// This collection holds references to entities owned by a
		// model. We use this to determine whether or not we can safely
		// destroy empty models.
		modelEntityRefsC: {global: true},

		// This collection is holds the parameters for model migrations.
		migrationsC: {
			global: true,
			indexes: []mgo.Index{{
				Key: []string{"model-uuid"},
			}},
		},

		// This collection tracks the progress of model migrations.
		migrationsStatusC: {global: true},

		// This collection records the model migrations which
		// are currently in progress. It is used to ensure that only
		// one model migration document exists per environment.
		migrationsActiveC: {global: true},

		// This collection holds user information that's not specific to any
		// one model.
		usersC: {
			global: true,
			indexes: []mgo.Index{{
				// TODO(thumper): schema change to remove this index.
				Key: []string{"name"},
			}},
		},

		// This collection holds the last time the user connected to the API server.
		userLastLoginC: {
			global:    true,
			rawAccess: true,
		},

		// This collection is used as a unique key restraint. The _id field is
		// a concatenation of multiple fields that form a compound index,
		// allowing us to ensure users cannot have the same name for two
		// different models at a time.
		usermodelnameC: {global: true},

		// This collection holds workload metrics reported by certain charms
		// for passing onward to other tools.
		metricsC: {global: true},

		// This collection holds persistent state for the metrics manager.
		metricsManagerC: {global: true},

		// This collection holds lease data, which is per-model, but is
		// not itself multi-model-aware; happily it will imminently be
		// deprecated in favour of the non-global leasesC below.
		// TODO(fwereade): drop leaseC entirely so can't use wrong const.
		leaseC: {global: true},

		// This collection was deprecated before multi-model support
		// was implemented.
		actionresultsC: {global: true},

		// This collection holds storage items for a macaroon bakery.
		bakeryStorageItemsC: {
			global:  true,
			indexes: bakerystorage.MongoIndexes(),
		},

		// -----------------

		// Local collections
		// =================

		// This collection is basically a standard SQL intersection table; it
		// references the global records of the users allowed access to a
		// given collection.
		modelUsersC: {},

		// This collection holds the last time the model user connected
		// to the model.
		modelUserLastConnectionC: {
			rawAccess: true,
		},

		// This collection contains governors that prevent certain kinds of
		// changes from being accepted.
		blocksC: {},

		// This collection is used for internal bookkeeping; certain complex
		// or tedious state changes are deferred by recording a cleanup doc
		// for later handling.
		cleanupsC: {},

		// This collection contains incrementing integers, subdivided by name,
		// to ensure various IDs aren't reused.
		sequenceC: {},

		// This collection holds lease data. It's currently only used to
		// implement service leadership, but is namespaced and available
		// for use by other clients in future.
		leasesC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "type"},
			}, {
				Key: []string{"model-uuid", "namespace"},
			}},
		},

		// -----

		// These collections hold information associated with services.
		charmsC:   {},
		servicesC: {},
		unitsC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "service"},
			}, {
				Key: []string{"model-uuid", "principal"},
			}, {
				Key: []string{"model-uuid", "machineid"},
			}},
		},
		minUnitsC: {},

		// This collection holds documents that indicate units which are queued
		// to be assigned to machines. It is used exclusively by the
		// AssignUnitWorker.
		assignUnitC: {},

		// meterStatusC is the collection used to store meter status information.
		meterStatusC:  {},
		settingsrefsC: {},
		relationsC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "endpoints.relationname"},
			}, {
				Key: []string{"model-uuid", "endpoints.servicename"},
			}},
		},
		relationScopesC: {},

		// -----

		// These collections hold information associated with machines.
		containerRefsC: {},
		instanceDataC:  {},
		machinesC:      {},
		rebootC:        {},
		sshHostKeysC:   {},

		// -----

		// These collections hold information associated with storage.
		blockDevicesC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "machineid"},
			}},
		},
		filesystemsC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "storageid"},
			}},
		},
		filesystemAttachmentsC: {},
		storageInstancesC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "owner"},
			}},
		},
		storageAttachmentsC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "storageid"},
			}, {
				Key: []string{"model-uuid", "unitid"},
			}},
		},
		volumesC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "storageid"},
			}},
		},
		volumeAttachmentsC: {},

		// -----

		// These collections hold information associated with networking.
		// TODO(dimitern): Remove the obsolete collections below once possible.
		legacyipaddressesC: {
			indexes: []mgo.Index{{
				Key: []string{"uuid"},
			}, {
				Key: []string{"model-uuid", "state"},
			}, {
				Key: []string{"model-uuid", "subnetid"},
			}},
		},
		// TODO(dimitern): End of obsolete networking collections.
		providerIDsC: {},
		spacesC:      {},
		subnetsC: {
			indexes: []mgo.Index{{
				Key:    []string{"providerid"},
				Unique: true,
				Sparse: true,
			}},
		},
		linkLayerDevicesC: {
			indexes: []mgo.Index{{
				Key:    []string{"providerid"},
				Unique: true,
				Sparse: true,
			}},
		},
		linkLayerDevicesRefsC: {},
		ipAddressesC: {
			indexes: []mgo.Index{{
				Key:    []string{"providerid"},
				Unique: true,
				Sparse: true,
			}},
		},
		endpointBindingsC: {},
		openedPortsC:      {},

		// -----

		// These collections hold information associated with actions.
		actionsC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "name"},
			}},
		},
		actionNotificationsC: {},

		// -----

		// TODO(ericsnow) Use a component-oriented registration mechanism...

		// This collection holds information associated with charm payloads.
		// See payload/persistence/mongo.go.
		"payloads": {},

		// This collection holds information associated with charm resources.
		// See resource/persistence/mongo.go.
		"resources": {},

		// -----

		// The remaining non-global collections share the property of being
		// relevant to multiple other kinds of entities, and are thus generally
		// indexed by globalKey(). This is unhelpfully named in this context --
		// it's meant to imply "global within an model", because it was
		// named before multi-env support.

		// This collection holds user annotations for various entities. They
		// shouldn't be written or interpreted by juju.
		annotationsC: {},

		// This collection in particular holds an astounding number of
		// different sorts of data: service config settings by charm version,
		// unit relation settings, model config, etc etc etc.
		settingsC: {},

		constraintsC:        {},
		storageConstraintsC: {},
		statusesC:           {},
		statusesHistoryC: {
			indexes: []mgo.Index{{
				Key: []string{"model-uuid", "globalkey"},
			}},
		},

		// This collection holds information about cloud image metadata.
		cloudimagemetadataC: {},

		// ----------------------

		// Raw-access collections
		// ======================

		// metrics; status-history; logs; ..?
	}
}

// These constants are used to avoid sprinkling the package with any more
// magic strings. If a collection deserves documentation, please document
// it in allCollections, above; and please keep this list sorted for easy
// inspection.
const (
	actionNotificationsC     = "actionnotifications"
	actionresultsC           = "actionresults"
	actionsC                 = "actions"
	annotationsC             = "annotations"
	assignUnitC              = "assignUnits"
	bakeryStorageItemsC      = "bakeryStorageItems"
	blockDevicesC            = "blockdevices"
	blocksC                  = "blocks"
	charmsC                  = "charms"
	cleanupsC                = "cleanups"
	cloudimagemetadataC      = "cloudimagemetadata"
	constraintsC             = "constraints"
	containerRefsC           = "containerRefs"
	controllersC             = "controllers"
	filesystemAttachmentsC   = "filesystemAttachments"
	filesystemsC             = "filesystems"
	guimetadataC             = "guimetadata"
	guisettingsC             = "guisettings"
	instanceDataC            = "instanceData"
	legacyipaddressesC       = "ipaddresses"
	leaseC                   = "lease"
	leasesC                  = "leases"
	machinesC                = "machines"
	meterStatusC             = "meterStatus"
	metricsC                 = "metrics"
	metricsManagerC          = "metricsmanager"
	minUnitsC                = "minunits"
	migrationsStatusC        = "migrations.status"
	migrationsActiveC        = "migrations.active"
	migrationsC              = "migrations"
	modelUserLastConnectionC = "modelUserLastConnection"
	modelUsersC              = "modelusers"
	modelsC                  = "models"
	modelEntityRefsC         = "modelEntityRefs"
	openedPortsC             = "openedPorts"
	providerIDsC             = "providerIDs"
	rebootC                  = "reboot"
	relationScopesC          = "relationscopes"
	relationsC               = "relations"
	restoreInfoC             = "restoreInfo"
	sequenceC                = "sequence"
	servicesC                = "services"
	endpointBindingsC        = "endpointbindings"
	settingsC                = "settings"
	settingsrefsC            = "settingsrefs"
	sshHostKeysC             = "sshhostkeys"
	spacesC                  = "spaces"
	statusesC                = "statuses"
	statusesHistoryC         = "statuseshistory"
	storageAttachmentsC      = "storageattachments"
	storageConstraintsC      = "storageconstraints"
	storageInstancesC        = "storageinstances"
	subnetsC                 = "subnets"
	linkLayerDevicesC        = "linklayerdevices"
	linkLayerDevicesRefsC    = "linklayerdevicesrefs"
	ipAddressesC             = "ip.addresses"
	toolsmetadataC           = "toolsmetadata"
	txnLogC                  = "txns.log"
	txnsC                    = "txns"
	unitsC                   = "units"
	upgradeInfoC             = "upgradeInfo"
	userLastLoginC           = "userLastLogin"
	usermodelnameC           = "usermodelname"
	usersC                   = "users"
	volumeAttachmentsC       = "volumeattachments"
	volumesC                 = "volumes"
	// "payloads" (see payload/persistence/mongo.go)
	// "resources" (see resource/persistence/mongo.go)
)
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"github.com/boltdb/bolt"
	"github.com/codegangsta/cli"
	kval "github.com/kval-access-language/kval-boltdb"
	"github.com/olekukonko/tablewriter"
)

func main() {
	var file string
	var bucket string
	var machineFriendly bool

	cli.AppHelpTemplate = `NAME:
  {{.Name}} - {{.Usage}}

VERSION:
  {{.Version}}

USAGE:
  {{.HelpName}} {{if .VisibleFlags}}[global options]{{end}}

GLOBAL OPTIONS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
AUTHOR:
  {{range .Authors}}{{ . }}{{end}}
`
	app := cli.NewApp()
	app.Name = "bolter"
	app.Usage = "view boltdb file in your terminal"
	app.Version = "1.0.0"
	app.Author = "Hasit Mistry"
	app.Email = "hasitnm@gmail.com"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "file, f",
			Usage:       "boltdb `FILE` to view",
			Destination: &file,
		},
		cli.StringFlag{
			Name:        "bucket, b",
			Usage:       "boltdb `BUCKET` to view",
			Destination: &bucket,
		},
		cli.BoolFlag{
			Name:        "machine, m",
			Usage:       "key=value format",
			Destination: &machineFriendly,
		},
	}
	app.Action = func(c *cli.Context) error {
		if file == "" {
			cli.ShowAppHelp(c)
			return nil
		}

		var i impl
		if machineFriendly {
			i = impl{fmt: &machineFormatter{}}
		} else {
			i = impl{fmt: &tableFormatter{}}
		}
		if _, err := os.Stat(file); os.IsNotExist(err) {
			log.Fatal(err)
			return err
		}
		i.initDB(file)
		defer i.DB.Close()

		i.readInput()

		return nil
	}
	app.Run(os.Args)
}

func (i *impl) readInput() {
	fmt.Fprint(os.Stdout, "DB Layout:\n\n")
	i.listBuckets()
	scanner := bufio.NewScanner(os.Stdin)
	fmt.Fprint(os.Stdout, "\nEnter bucket to explore (q to quit, m to print):\n\n")
	for scanner.Scan() {
		bucket := scanner.Text()
		fmt.Fprintln(os.Stdout, "")
		switch(bucket) {
		case "q":
			fallthrough
		case "quit":
			return
		case "":
			i.listBuckets()
		default:
			i.listBucketItems(bucket)
		}
	}
}

type formatter interface {
	DumpBuckets([]bucket)
	DumpBucketItems(string, []item)
}

type impl struct {
	KV  kval.Kvalboltdb
	DB  *bolt.DB
	fmt formatter
	loc string		// where we are in the structure
}

type item struct {
	Key   string `json:"Key"`
	Value string `json:"Value"`
}

type bucket struct {
	Name string `json:"Name"`
}

func (i *impl) initDB(file string) {
	var err error
	// Read-only permission, should be equiv. bolt.Open(file, 0400, nil)
	i.KV, err = kval.Connect(file)
	i.DB = kval.GetBolt(i.KV)
	if err != nil {
		log.Fatal(err)
	}
}

func (i *impl) updateLoc(bucket string) string {
	if i.loc == "" {
		i.loc = "GET " + bucket
	} else {
		i.loc = i.loc + " >> " + bucket
	}
	return i.loc
}

func (i *impl) listBucketItems(bucket string) {
	items := []item{}
	getItems := i.updateLoc(bucket)
	fmt.Fprintf(os.Stderr, "Query: " + i.loc + "\n\n")
	res, err := kval.Query(i.KV, getItems)
	if err != nil {
		if err.Error() != "Cannot GOTO bucket, bucket not found" {
			log.Fatal(err)
		} else {
			fmt.Fprintln(os.Stderr, "Bucket not found")
			fmt.Println(getItems)
		}
	}
	for k, v := range res.Result {
		if v == kval.Nestedbucket {
			k = k + "*"
			v = ""
		}
		items = append(items, item{Key: string(k), Value: string(v)})
	}

	i.fmt.DumpBucketItems(bucket, items)
}

func (i *impl) listBuckets() {
	buckets := []bucket{}
	err := i.DB.View(func(tx *bolt.Tx) error {
		return tx.ForEach(func(bucketname []byte, _ *bolt.Bucket) error {
			buckets = append(buckets, bucket{Name: string(bucketname)})
			return nil
		})
	})
	if err != nil {
		log.Fatal(err)
	}
	i.fmt.DumpBuckets(buckets)
}

type tableFormatter struct{}

func (tf tableFormatter) DumpBuckets(buckets []bucket) {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Buckets"})
	for _, b := range buckets {
		row := []string{b.Name}
		table.Append(row)
	}
	table.Render()
	fmt.Println()
}

func (tf tableFormatter) DumpBucketItems(bucket string, items []item) {
	fmt.Printf("Bucket: %s\n", bucket)
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Key", "Value"})
	for _, item := range items {
		row := []string{item.Key, item.Value}
		table.Append(row)
	}
	table.Render()
	fmt.Println()
}

type machineFormatter struct{}

func (mf machineFormatter) DumpBuckets(buckets []bucket) {
	for _, b := range buckets {
		fmt.Println(b.Name)
	}
}

func (mf machineFormatter) DumpBucketItems(_ string, items []item) {
	for _, item := range items {
		fmt.Printf("%s=%s\n", item.Key, item.Value)
	}
}
<commit_msg>[feature] Remove machine readable.<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"github.com/boltdb/bolt"
	"github.com/codegangsta/cli"
	kval "github.com/kval-access-language/kval-boltdb"
	"github.com/olekukonko/tablewriter"
)

func main() {
	var file string
	var bucket string

	cli.AppHelpTemplate = `NAME:
  {{.Name}} - {{.Usage}}

VERSION:
  {{.Version}}

USAGE:
  {{.HelpName}} {{if .VisibleFlags}}[global options]{{end}}

GLOBAL OPTIONS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
AUTHOR:
  {{range .Authors}}{{ . }}{{end}}
`
	app := cli.NewApp()
	app.Name = "bolter"
	app.Usage = "view boltdb file in your terminal"
	app.Version = "1.0.0"
	app.Author = "Hasit Mistry"
	app.Email = "hasitnm@gmail.com"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "file, f",
			Usage:       "boltdb `FILE` to view",
			Destination: &file,
		},
		cli.StringFlag{
			Name:        "bucket, b",
			Usage:       "boltdb `BUCKET` to view",
			Destination: &bucket,
		},
	}
	app.Action = func(c *cli.Context) error {
		if file == "" {
			cli.ShowAppHelp(c)
			return nil
		}

		var i impl
		i = impl{fmt: &tableFormatter{}}
		if _, err := os.Stat(file); os.IsNotExist(err) {
			log.Fatal(err)
			return err
		}
		i.initDB(file)
		defer i.DB.Close()

		i.readInput()

		return nil
	}
	app.Run(os.Args)
}

func (i *impl) readInput() {
	fmt.Fprint(os.Stdout, "DB Layout:\n\n")
	i.listBuckets()
	scanner := bufio.NewScanner(os.Stdin)
	fmt.Fprint(os.Stdout, "\nEnter bucket to explore (q to quit, m to print):\n\n")
	for scanner.Scan() {
		bucket := scanner.Text()
		fmt.Fprintln(os.Stdout, "")
		switch(bucket) {
		case "q":
			fallthrough
		case "quit":
			return
		case "":
			i.listBuckets()
		default:
			i.listBucketItems(bucket)
		}
	}
}

type formatter interface {
	DumpBuckets([]bucket)
	DumpBucketItems(string, []item)
}

type impl struct {
	KV  kval.Kvalboltdb
	DB  *bolt.DB
	fmt formatter
	loc string		// where we are in the structure
}

type item struct {
	Key   string `json:"Key"`
	Value string `json:"Value"`
}

type bucket struct {
	Name string `json:"Name"`
}

func (i *impl) initDB(file string) {
	var err error
	// Read-only permission, should be equiv. bolt.Open(file, 0400, nil)
	i.KV, err = kval.Connect(file)
	i.DB = kval.GetBolt(i.KV)
	if err != nil {
		log.Fatal(err)
	}
}

func (i *impl) updateLoc(bucket string) string {
	if i.loc == "" {
		i.loc = "GET " + bucket
	} else {
		i.loc = i.loc + " >> " + bucket
	}
	return i.loc
}

func (i *impl) listBucketItems(bucket string) {
	items := []item{}
	getItems := i.updateLoc(bucket)
	fmt.Fprintf(os.Stderr, "Query: " + i.loc + "\n\n")
	res, err := kval.Query(i.KV, getItems)
	if err != nil {
		if err.Error() != "Cannot GOTO bucket, bucket not found" {
			log.Fatal(err)
		} else {
			fmt.Fprintln(os.Stderr, "Bucket not found")
			fmt.Println(getItems)
		}
	}
	for k, v := range res.Result {
		if v == kval.Nestedbucket {
			k = k + "*"
			v = ""
		}
		items = append(items, item{Key: string(k), Value: string(v)})
	}

	i.fmt.DumpBucketItems(bucket, items)
}

func (i *impl) listBuckets() {
	buckets := []bucket{}
	err := i.DB.View(func(tx *bolt.Tx) error {
		return tx.ForEach(func(bucketname []byte, _ *bolt.Bucket) error {
			buckets = append(buckets, bucket{Name: string(bucketname)})
			return nil
		})
	})
	if err != nil {
		log.Fatal(err)
	}
	i.fmt.DumpBuckets(buckets)
}

type tableFormatter struct{}

func (tf tableFormatter) DumpBuckets(buckets []bucket) {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Buckets"})
	for _, b := range buckets {
		row := []string{b.Name}
		table.Append(row)
	}
	table.Render()
	fmt.Println()
}

func (tf tableFormatter) DumpBucketItems(bucket string, items []item) {
	fmt.Printf("Bucket: %s\n", bucket)
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Key", "Value"})
	for _, item := range items {
		row := []string{item.Key, item.Value}
		table.Append(row)
	}
	table.Render()
	fmt.Println()
}

<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/log"
	"github.com/prometheus/procfs"
)

var (
	// command line parameters
	listenAddress     = flag.String("web.listen-address", "9256", "Address on which to expose metrics and web interface.")
	metricsPath       = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
	processNamePrefix = flag.String("fluentd.process_name_prefix", "", "fluentd's process_name prefix.")

	processNameRegex = regexp.MustCompile(`\s/usr/sbin/td-agent\s*`)
	tdAgentPathRegex       = regexp.MustCompile("\\s" + strings.Replace(tdAgentLaunchCommand, " ", "\\s", -1) + "(.+)?\\s*")
	configFileNameRegex    = regexp.MustCompile(`\s(-c|--config)\s.*/(.+)\.conf\s*`)
	processNamePrefixRegex = regexp.MustCompile(`\sworker:(.+)?\s*`)
)

const (
	// Can't use '-' for the metric name
	namespace            = "td_agent"
	tdAgentLaunchCommand = "/opt/td-agent/embedded/bin/ruby /usr/sbin/td-agent "
)

type Exporter struct {
	mutex sync.RWMutex

	scrapeFailures prometheus.Counter
	cpuTime        *prometheus.CounterVec
	virtualMemory  *prometheus.GaugeVec
	residentMemory *prometheus.GaugeVec
	tdAgentUp      prometheus.Gauge
}

func NewExporter() *Exporter {
	return &Exporter{
		scrapeFailures: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "exporter_scrape_failures_total",
			Help:      "Number of errors while scraping td-agent.",
		}),
		cpuTime: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: namespace,
				Name:      "cpu_time",
				Help:      "td-agent cpu time",
			},
			[]string{"id"},
		),
		virtualMemory: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: namespace,
				Name:      "virtual_memory_usage",
				Help:      "td-agent virtual memory usage",
			},
			[]string{"id"},
		),
		residentMemory: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: namespace,
				Name:      "resident_memory_usage",
				Help:      "td-agent resident memory usage",
			},
			[]string{"id"},
		),
		tdAgentUp: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "up",
			Help:      "the td-agent processes",
		}),
	}
}

func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	e.scrapeFailures.Describe(ch)
	e.cpuTime.Describe(ch)
	e.virtualMemory.Describe(ch)
	e.residentMemory.Describe(ch)
	e.tdAgentUp.Describe(ch)
}

func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	// To protect metrics from concurrent collects.
	e.mutex.Lock()
	defer e.mutex.Unlock()
	e.collect(ch)
}

func (e *Exporter) collect(ch chan<- prometheus.Metric) {
	e.tdAgentUp.Set(0)

	ids, err := e.resolveTdAgentId()
	if err != nil {
		e.scrapeFailures.Inc()
		e.scrapeFailures.Collect(ch)
		return
	}

	log.Debugf("td-agent ids = %v", ids)

	processes := 0
	for id, command := range ids {
		log.Debugf("td-agent id = %s", id)

		procStat, err := e.getProcStat(id, command)
		if err != nil {
			e.scrapeFailures.Inc()
			e.scrapeFailures.Collect(ch)
			continue
		}

		e.cpuTime.WithLabelValues(id).Add(procStat.CPUTime())
		e.virtualMemory.WithLabelValues(id).Set(float64(procStat.VirtualMemory()))
		e.residentMemory.WithLabelValues(id).Set(float64(procStat.ResidentMemory()))

		processes++
	}

	e.tdAgentUp.Set(float64(processes))

	e.cpuTime.Collect(ch)
	e.virtualMemory.Collect(ch)
	e.residentMemory.Collect(ch)
	e.tdAgentUp.Collect(ch)
}

func (e *Exporter) resolveTdAgentId() (map[string]string, error) {
	lines, err := e.execPsCommand()
	if err != nil {
		return nil, err
	}
	filtered := e.filter(strings.Split(lines, "\n"))

	log.Debugf("filtered = %s", filtered)

	var ids map[string]string
	if *processNamePrefix == "" {
		ids = e.resolveTdAgentIdWithConfigFileName(filtered)
	} else {
		ids, err = e.resolveTdAgentIdWithProcessNamePrefix(filtered)
		if err != nil {
			return nil, err
		}
	}

	return ids, nil
}

func (e *Exporter) execPsCommand() (string, error) {
	ps, err := exec.Command("ps", "-C", "ruby", "-f").Output()
	if err != nil {
		log.Error(err)
		return "", err
	}

	return string(ps), nil
}

func (e *Exporter) filter(lines []string) []string {
	var filtered []string
	for _, line := range lines {
		log.Debugf("line = %s", line)

		if *processNamePrefix == "" {
			if processNameRegex.MatchString(line) {
				filtered = append(filtered, line)
			}
		} else {
			groups := processNamePrefixRegex.FindStringSubmatch(line)
			if len(groups) > 0 {
				filtered = append(filtered, line)
			}
		}
	}

	return filtered
}

func (e *Exporter) resolveTdAgentIdWithConfigFileName(lines []string) map[string]string {
	ids := make(map[string]string)
	for _, line := range lines {
		log.Debugf("resolveTdAgentIdWithConfigFileName line = %s", line)

		groupsKey := configFileNameRegex.FindStringSubmatch(line)
		var key string
		if len(groupsKey) == 0 {
			// doesn't use config file
			key = "default"
		} else {
			key = groupsKey[2]
		}

		var value string
		groupsValue := tdAgentPathRegex.FindStringSubmatch(line)
		if len(groupsValue) > 0 {
			value = tdAgentLaunchCommand + groupsValue[1]
		} else {
			value = ""
		}

		if _, exist := ids[key]; !exist {
			ids[key] = value
		}
	}

	return ids
}

func (e *Exporter) resolveTdAgentIdWithProcessNamePrefix(lines []string) (map[string]string, error) {
	ids := make(map[string]string)
	for _, line := range lines {
		log.Debugf("resolveTdAgentIdWithProcessNamePrefix line = %s", line)

		groupsKey := processNamePrefixRegex.FindStringSubmatch(line)
		if len(groupsKey) == 0 {
			err := fmt.Errorf("Process not found. fluentd.process_name_prefix = `%s`", processNamePrefixRegex)
			log.Error(err)
			return nil, err
		}

		key := groupsKey[1]

		log.Debugf("key = %s", key)

		var value string
		groupsValue := tdAgentPathRegex.FindStringSubmatch(line)
		if len(groupsValue) > 0 {
			value = tdAgentLaunchCommand + groupsValue[1]
		} else {
			value = ""
		}

		if _, exist := ids[key]; !exist {
			ids[key] = value
		}
	}

	return ids, nil
}

func (e *Exporter) getProcStat(tdAgentId string, tdAgentCommand string) (procfs.ProcStat, error) {
	procfsPath := procfs.DefaultMountPoint
	fs, err := procfs.NewFS(procfsPath)
	if err != nil {
		log.Error(err)
		return procfs.ProcStat{}, err
	}

	targetPid, err := e.resolveTargetPid(tdAgentId, tdAgentCommand)
	if err != nil {
		return procfs.ProcStat{}, err
	}

	log.Debugf("targetPid = %v", targetPid)

	proc, err := fs.NewProc(targetPid)
	if err != nil {
		log.Error(err)
		return procfs.ProcStat{}, err
	}

	procStat, err := proc.NewStat()
	if err != nil {
		log.Error(err)
		return procfs.ProcStat{}, err
	}

	return procStat, nil
}

func (e *Exporter) resolveTargetPid(tdAgentId string, tdAgentCommand string) (int, error) {
	var pgrepArg string

	if *processNamePrefix == "" {
		if strings.HasSuffix(tdAgentId, ".conf") {
			pgrepArg = tdAgentId + ".conf"
		} else {
			pgrepArg = tdAgentCommand
		}
	} else {
		pgrepArg = ":" + tdAgentId
	}

	log.Debugf("pgrep arg = [%s]", pgrepArg)

	out, err := exec.Command("pgrep", "-n", "-f", strings.Replace(pgrepArg, " ", "\\ ", -1)).Output()
	if err != nil {
		log.Error(err)
		return 0, err
	}

	targetPid, err := strconv.Atoi(strings.TrimSpace(string(out)))
	if err != nil {
		log.Error(err)
		return 0, err
	}

	return targetPid, nil
}

func main() {
	flag.Parse()

	exporter := NewExporter()
	prometheus.MustRegister(exporter)

	log.Infof("Starting Server: %s", *listenAddress)
	http.Handle(*metricsPath, promhttp.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
		<title>td-agent Exporter</title>

td-agent Exporter v` + version + `

<a href="` + *metricsPath + `">Metrics

		`))
	})

	log.Fatal(http.ListenAndServe(":"+*listenAddress, nil))
}
<commit_msg>apply `go fmt`<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/log"
	"github.com/prometheus/procfs"
)

var (
	// command line parameters
	listenAddress     = flag.String("web.listen-address", "9256", "Address on which to expose metrics and web interface.")
	metricsPath       = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
	processNamePrefix = flag.String("fluentd.process_name_prefix", "", "fluentd's process_name prefix.")

	processNameRegex       = regexp.MustCompile(`\s/usr/sbin/td-agent\s*`)
	tdAgentPathRegex       = regexp.MustCompile("\\s" + strings.Replace(tdAgentLaunchCommand, " ", "\\s", -1) + "(.+)?\\s*")
	configFileNameRegex    = regexp.MustCompile(`\s(-c|--config)\s.*/(.+)\.conf\s*`)
	processNamePrefixRegex = regexp.MustCompile(`\sworker:(.+)?\s*`)
)

const (
	// Can't use '-' for the metric name
	namespace            = "td_agent"
	tdAgentLaunchCommand = "/opt/td-agent/embedded/bin/ruby /usr/sbin/td-agent "
)

type Exporter struct {
	mutex sync.RWMutex

	scrapeFailures prometheus.Counter
	cpuTime        *prometheus.CounterVec
	virtualMemory  *prometheus.GaugeVec
	residentMemory *prometheus.GaugeVec
	tdAgentUp      prometheus.Gauge
}

func NewExporter() *Exporter {
	return &Exporter{
		scrapeFailures: prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: namespace,
			Name:      "exporter_scrape_failures_total",
			Help:      "Number of errors while scraping td-agent.",
		}),
		cpuTime: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: namespace,
				Name:      "cpu_time",
				Help:      "td-agent cpu time",
			},
			[]string{"id"},
		),
		virtualMemory: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: namespace,
				Name:      "virtual_memory_usage",
				Help:      "td-agent virtual memory usage",
			},
			[]string{"id"},
		),
		residentMemory: prometheus.NewGaugeVec(
			prometheus.GaugeOpts{
				Namespace: namespace,
				Name:      "resident_memory_usage",
				Help:      "td-agent resident memory usage",
			},
			[]string{"id"},
		),
		tdAgentUp: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      "up",
			Help:      "the td-agent processes",
		}),
	}
}

func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	e.scrapeFailures.Describe(ch)
	e.cpuTime.Describe(ch)
	e.virtualMemory.Describe(ch)
	e.residentMemory.Describe(ch)
	e.tdAgentUp.Describe(ch)
}

func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
	// To protect metrics from concurrent collects.
	e.mutex.Lock()
	defer e.mutex.Unlock()
	e.collect(ch)
}

func (e *Exporter) collect(ch chan<- prometheus.Metric) {
	e.tdAgentUp.Set(0)

	ids, err := e.resolveTdAgentId()
	if err != nil {
		e.scrapeFailures.Inc()
		e.scrapeFailures.Collect(ch)
		return
	}

	log.Debugf("td-agent ids = %v", ids)

	processes := 0
	for id, command := range ids {
		log.Debugf("td-agent id = %s", id)

		procStat, err := e.getProcStat(id, command)
		if err != nil {
			e.scrapeFailures.Inc()
			e.scrapeFailures.Collect(ch)
			continue
		}

		e.cpuTime.WithLabelValues(id).Add(procStat.CPUTime())
		e.virtualMemory.WithLabelValues(id).Set(float64(procStat.VirtualMemory()))
		e.residentMemory.WithLabelValues(id).Set(float64(procStat.ResidentMemory()))

		processes++
	}

	e.tdAgentUp.Set(float64(processes))

	e.cpuTime.Collect(ch)
	e.virtualMemory.Collect(ch)
	e.residentMemory.Collect(ch)
	e.tdAgentUp.Collect(ch)
}

func (e *Exporter) resolveTdAgentId() (map[string]string, error) {
	lines, err := e.execPsCommand()
	if err != nil {
		return nil, err
	}
	filtered := e.filter(strings.Split(lines, "\n"))

	log.Debugf("filtered = %s", filtered)

	var ids map[string]string
	if *processNamePrefix == "" {
		ids = e.resolveTdAgentIdWithConfigFileName(filtered)
	} else {
		ids, err = e.resolveTdAgentIdWithProcessNamePrefix(filtered)
		if err != nil {
			return nil, err
		}
	}

	return ids, nil
}

func (e *Exporter) execPsCommand() (string, error) {
	ps, err := exec.Command("ps", "-C", "ruby", "-f").Output()
	if err != nil {
		log.Error(err)
		return "", err
	}

	return string(ps), nil
}

func (e *Exporter) filter(lines []string) []string {
	var filtered []string
	for _, line := range lines {
		log.Debugf("line = %s", line)

		if *processNamePrefix == "" {
			if processNameRegex.MatchString(line) {
				filtered = append(filtered, line)
			}
		} else {
			groups := processNamePrefixRegex.FindStringSubmatch(line)
			if len(groups) > 0 {
				filtered = append(filtered, line)
			}
		}
	}

	return filtered
}

func (e *Exporter) resolveTdAgentIdWithConfigFileName(lines []string) map[string]string {
	ids := make(map[string]string)
	for _, line := range lines {
		log.Debugf("resolveTdAgentIdWithConfigFileName line = %s", line)

		groupsKey := configFileNameRegex.FindStringSubmatch(line)
		var key string
		if len(groupsKey) == 0 {
			// doesn't use config file
			key = "default"
		} else {
			key = groupsKey[2]
		}

		var value string
		groupsValue := tdAgentPathRegex.FindStringSubmatch(line)
		if len(groupsValue) > 0 {
			value = tdAgentLaunchCommand + groupsValue[1]
		} else {
			value = ""
		}

		if _, exist := ids[key]; !exist {
			ids[key] = value
		}
	}

	return ids
}

func (e *Exporter) resolveTdAgentIdWithProcessNamePrefix(lines []string) (map[string]string, error) {
	ids := make(map[string]string)
	for _, line := range lines {
		log.Debugf("resolveTdAgentIdWithProcessNamePrefix line = %s", line)

		groupsKey := processNamePrefixRegex.FindStringSubmatch(line)
		if len(groupsKey) == 0 {
			err := fmt.Errorf("Process not found. fluentd.process_name_prefix = `%s`", processNamePrefixRegex)
			log.Error(err)
			return nil, err
		}

		key := groupsKey[1]

		log.Debugf("key = %s", key)

		var value string
		groupsValue := tdAgentPathRegex.FindStringSubmatch(line)
		if len(groupsValue) > 0 {
			value = tdAgentLaunchCommand + groupsValue[1]
		} else {
			value = ""
		}

		if _, exist := ids[key]; !exist {
			ids[key] = value
		}
	}

	return ids, nil
}

func (e *Exporter) getProcStat(tdAgentId string, tdAgentCommand string) (procfs.ProcStat, error) {
	procfsPath := procfs.DefaultMountPoint
	fs, err := procfs.NewFS(procfsPath)
	if err != nil {
		log.Error(err)
		return procfs.ProcStat{}, err
	}

	targetPid, err := e.resolveTargetPid(tdAgentId, tdAgentCommand)
	if err != nil {
		return procfs.ProcStat{}, err
	}

	log.Debugf("targetPid = %v", targetPid)

	proc, err := fs.NewProc(targetPid)
	if err != nil {
		log.Error(err)
		return procfs.ProcStat{}, err
	}

	procStat, err := proc.NewStat()
	if err != nil {
		log.Error(err)
		return procfs.ProcStat{}, err
	}

	return procStat, nil
}

func (e *Exporter) resolveTargetPid(tdAgentId string, tdAgentCommand string) (int, error) {
	var pgrepArg string

	if *processNamePrefix == "" {
		if strings.HasSuffix(tdAgentId, ".conf") {
			pgrepArg = tdAgentId + ".conf"
		} else {
			pgrepArg = tdAgentCommand
		}
	} else {
		pgrepArg = ":" + tdAgentId
	}

	log.Debugf("pgrep arg = [%s]", pgrepArg)

	out, err := exec.Command("pgrep", "-n", "-f", strings.Replace(pgrepArg, " ", "\\ ", -1)).Output()
	if err != nil {
		log.Error(err)
		return 0, err
	}

	targetPid, err := strconv.Atoi(strings.TrimSpace(string(out)))
	if err != nil {
		log.Error(err)
		return 0, err
	}

	return targetPid, nil
}

func main() {
	flag.Parse()

	exporter := NewExporter()
	prometheus.MustRegister(exporter)

	log.Infof("Starting Server: %s", *listenAddress)
	http.Handle(*metricsPath, promhttp.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`
		<title>td-agent Exporter</title>

td-agent Exporter v` + version + `

<a href="` + *metricsPath + `">Metrics

		`))
	})

	log.Fatal(http.ListenAndServe(":"+*listenAddress, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"github.com/BurntSushi/toml"
	"github.com/kelseyhightower/confd/log"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"text/template"
)

// TemplateResourceConfig holds the parsed template resource.
type TemplateResourceConfig struct {
	TemplateResource TemplateResource `toml:"template"`
}

// A fileInfo describes a configuration file and is returned by fileStat.
type fileInfo struct {
	Uid  uint32
	Gid  uint32
	Mode uint32
	Md5  string
}

// TemplateResource is the representation of a parsed template resource.
type TemplateResource struct {
	Dest      string
	Gid       int
	Keys      []string
	Mode      string
	Uid       int
	ReloadCmd string `toml:"reload_cmd"`
	CheckCmd  string `toml:"check_cmd"`
	StageFile *os.File
	Src       string
	Vars      map[string]interface{}
}

// setVars sets the Vars for template resource.
func (t *TemplateResource) setVars() error {
	c, err := newEtcdClient(EtcdNodes(), ClientCert(), ClientKey())
	if err != nil {
		return err
	}
	t.Vars, err = getValues(c, Prefix(), t.Keys)
	if err != nil {
		return err
	}
	return nil
}

// createStageFile stages the src configuration file by processing the src
// template and setting the desired owner, group, and mode. It also sets the
// StageFile for the template resource.
// It returns an error if any.
func (t *TemplateResource) createStageFile() error {
	t.Src = filepath.Join(TemplateDir(), t.Src)
	if !isFileExist(t.Src) {
		return errors.New("Missing template: " + t.Src)
	}
	temp, err := ioutil.TempFile("", "")
	if err != nil {
		os.Remove(temp.Name())
		return err
	}
	tmpl := template.Must(template.ParseFiles(t.Src))
	if err = tmpl.Execute(temp, t.Vars); err != nil {
		return err
	}
	// Set the owner, group, and mode on the stage file now to make it easier to
	// compare against the destination configuration file later.
	mode, _ := strconv.ParseUint(t.Mode, 0, 32)
	os.Chmod(temp.Name(), os.FileMode(mode))
	os.Chown(temp.Name(), t.Uid, t.Gid)
	t.StageFile = temp
	return nil
}

// sync compares the staged and dest config files and attempts to sync them
// if they differ. sync will run a config check command if set before
// overwriting the target config file. Finally, sync will run a reload command
// if set to have the application or service pick up the changes.
// It returns an error if any.
func (t *TemplateResource) sync() error {
	staged := t.StageFile.Name()
	defer os.Remove(staged)
	err, ok := sameConfig(staged, t.Dest)
	if err != nil {
		log.Error(err.Error())
	}
	if !ok {
		log.Info(t.Dest + " not in sync")
		if t.CheckCmd != "" {
			if err := t.check(); err != nil {
				return errors.New("Config check failed: " + err.Error())
			}
		}
		os.Rename(staged, t.Dest)
		if t.ReloadCmd != "" {
			if err := t.reload(); err != nil {
				return err
			}
		}
	} else {
		log.Info(t.Dest + " in sync")
	}
	return nil
}

// check executes the check command to validate the staged config file. The
// command is modified so that any references to src template are substituted
// with a string representing the full path of the staged file. This allows the
// check to be run on the staged file before overwriting the destination config
// file.
// It returns nil if the check command returns 0 and there are no other errors.
func (t *TemplateResource) check() error {
	var cmdBuffer bytes.Buffer
	data := make(map[string]string)
	data["src"] = t.StageFile.Name()
	tmpl, err := template.New("checkcmd").Parse(t.CheckCmd)
	if err != nil {
		return err
	}
	if err := tmpl.Execute(&cmdBuffer, data); err != nil {
		return err
	}
	log.Debug("Running " + cmdBuffer.String())
	c := exec.Command("/bin/sh", "-c", cmdBuffer.String())
	if err := c.Run(); err != nil {
		return err
	}
	return nil
}

// reload executes the reload command.
// It returns nil if the reload command returns 0.
func (t *TemplateResource) reload() error {
	log.Debug("Running " + t.ReloadCmd)
	c := exec.Command("/bin/sh", "-c", t.ReloadCmd)
	if err := c.Run(); err != nil {
		return err
	}
	return nil
}

// process is a convenience function that wraps calls to the three main tasks
// required to keep local configuration files in sync. First we gather vars
// from etcd, then we stage a candidate configuration file, and finally sync
// things up.
// It returns an error if any.
func (t *TemplateResource) process() error {
	if err := t.setVars(); err != nil {
		return err
	}
	if err := t.createStageFile(); err != nil {
		return err
	}
	if err := t.sync(); err != nil {
		return err
	}
	return nil
}

// ProcessTemplateResources is a convenience function that loads all the
// template resources and processes them serially. Called from main.
// It return an error if any.
func ProcessTemplateResources() error {
	paths, err := filepath.Glob(filepath.Join(ConfigDir(), "*toml"))
	if err != nil {
		return err
	}
	for _, p := range paths {
		var tc *TemplateResourceConfig
		_, err := toml.DecodeFile(p, &tc)
		if err != nil {
			return err
		}
		if err := tc.TemplateResource.process(); err != nil {
			return err
		}
	}
	return nil
}

// fileStat return a fileInfo describing the named file.
func fileStat(name string) (fi fileInfo, err error) {
	if isFileExist(name) {
		f, err := os.Open(name)
		defer f.Close()
		if err != nil {
			return fi, err
		}
		stats, _ := f.Stat()
		fi.Uid = stats.Sys().(*syscall.Stat_t).Uid
		fi.Gid = stats.Sys().(*syscall.Stat_t).Gid
		fi.Mode = stats.Sys().(*syscall.Stat_t).Mode
		h := md5.New()
		io.Copy(h, f)
		fi.Md5 = fmt.Sprintf("%x", h.Sum(nil))
		return fi, nil
	} else {
		return fi, errors.New("File not found")
	}
}

// sameConfig reports whether src and dest config files are equal.
// Two config files are equal when they have the same file contents and
// Unix permissions. The owner, group, and mode must match.
// It return false in other cases.
func sameConfig(src, dest string) (error, bool) {
	if !isFileExist(dest) {
		return nil, false
	}
	d, err := fileStat(dest)
	if err != nil {
		return err, false
	}
	s, err := fileStat(src)
	if err != nil {
		return err, false
	}
	if d.Uid != s.Uid || d.Gid != s.Gid || d.Mode != s.Mode || d.Md5 != s.Md5 {
		return nil, false
	}
	return nil, true
}
<commit_msg>Add a note regarding the non portable way we gather the uid, gid, and mode of a file<commit_after>package main

import (
	"bytes"
	"crypto/md5"
	"errors"
	"fmt"
	"github.com/BurntSushi/toml"
	"github.com/kelseyhightower/confd/log"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"text/template"
)

// TemplateResourceConfig holds the parsed template resource.
type TemplateResourceConfig struct {
	TemplateResource TemplateResource `toml:"template"`
}

// A fileInfo describes a configuration file and is returned by fileStat.
type fileInfo struct {
	Uid  uint32
	Gid  uint32
	Mode uint32
	Md5  string
}

// TemplateResource is the representation of a parsed template resource.
type TemplateResource struct {
	Dest      string
	Gid       int
	Keys      []string
	Mode      string
	Uid       int
	ReloadCmd string `toml:"reload_cmd"`
	CheckCmd  string `toml:"check_cmd"`
	StageFile *os.File
	Src       string
	Vars      map[string]interface{}
}

// setVars sets the Vars for template resource.
func (t *TemplateResource) setVars() error {
	c, err := newEtcdClient(EtcdNodes(), ClientCert(), ClientKey())
	if err != nil {
		return err
	}
	t.Vars, err = getValues(c, Prefix(), t.Keys)
	if err != nil {
		return err
	}
	return nil
}

// createStageFile stages the src configuration file by processing the src
// template and setting the desired owner, group, and mode. It also sets the
// StageFile for the template resource.
// It returns an error if any.
func (t *TemplateResource) createStageFile() error {
	t.Src = filepath.Join(TemplateDir(), t.Src)
	if !isFileExist(t.Src) {
		return errors.New("Missing template: " + t.Src)
	}
	temp, err := ioutil.TempFile("", "")
	if err != nil {
		os.Remove(temp.Name())
		return err
	}
	tmpl := template.Must(template.ParseFiles(t.Src))
	if err = tmpl.Execute(temp, t.Vars); err != nil {
		return err
	}
	// Set the owner, group, and mode on the stage file now to make it easier to
	// compare against the destination configuration file later.
	mode, _ := strconv.ParseUint(t.Mode, 0, 32)
	os.Chmod(temp.Name(), os.FileMode(mode))
	os.Chown(temp.Name(), t.Uid, t.Gid)
	t.StageFile = temp
	return nil
}

// sync compares the staged and dest config files and attempts to sync them
// if they differ. sync will run a config check command if set before
// overwriting the target config file. Finally, sync will run a reload command
// if set to have the application or service pick up the changes.
// It returns an error if any.
func (t *TemplateResource) sync() error {
	staged := t.StageFile.Name()
	defer os.Remove(staged)
	err, ok := sameConfig(staged, t.Dest)
	if err != nil {
		log.Error(err.Error())
	}
	if !ok {
		log.Info(t.Dest + " not in sync")
		if t.CheckCmd != "" {
			if err := t.check(); err != nil {
				return errors.New("Config check failed: " + err.Error())
			}
		}
		os.Rename(staged, t.Dest)
		if t.ReloadCmd != "" {
			if err := t.reload(); err != nil {
				return err
			}
		}
	} else {
		log.Info(t.Dest + " in sync")
	}
	return nil
}

// check executes the check command to validate the staged config file. The
// command is modified so that any references to src template are substituted
// with a string representing the full path of the staged file. This allows the
// check to be run on the staged file before overwriting the destination config
// file.
// It returns nil if the check command returns 0 and there are no other errors.
func (t *TemplateResource) check() error {
	var cmdBuffer bytes.Buffer
	data := make(map[string]string)
	data["src"] = t.StageFile.Name()
	tmpl, err := template.New("checkcmd").Parse(t.CheckCmd)
	if err != nil {
		return err
	}
	if err := tmpl.Execute(&cmdBuffer, data); err != nil {
		return err
	}
	log.Debug("Running " + cmdBuffer.String())
	c := exec.Command("/bin/sh", "-c", cmdBuffer.String())
	if err := c.Run(); err != nil {
		return err
	}
	return nil
}

// reload executes the reload command.
// It returns nil if the reload command returns 0.
func (t *TemplateResource) reload() error {
	log.Debug("Running " + t.ReloadCmd)
	c := exec.Command("/bin/sh", "-c", t.ReloadCmd)
	if err := c.Run(); err != nil {
		return err
	}
	return nil
}

// process is a convenience function that wraps calls to the three main tasks
// required to keep local configuration files in sync. First we gather vars
// from etcd, then we stage a candidate configuration file, and finally sync
// things up.
// It returns an error if any.
func (t *TemplateResource) process() error {
	if err := t.setVars(); err != nil {
		return err
	}
	if err := t.createStageFile(); err != nil {
		return err
	}
	if err := t.sync(); err != nil {
		return err
	}
	return nil
}

// ProcessTemplateResources is a convenience function that loads all the
// template resources and processes them serially. Called from main.
// It return an error if any.
func ProcessTemplateResources() error {
	paths, err := filepath.Glob(filepath.Join(ConfigDir(), "*toml"))
	if err != nil {
		return err
	}
	for _, p := range paths {
		var tc *TemplateResourceConfig
		_, err := toml.DecodeFile(p, &tc)
		if err != nil {
			return err
		}
		if err := tc.TemplateResource.process(); err != nil {
			return err
		}
	}
	return nil
}

// fileStat return a fileInfo describing the named file.
func fileStat(name string) (fi fileInfo, err error) {
	if isFileExist(name) {
		f, err := os.Open(name)
		defer f.Close()
		if err != nil {
			return fi, err
		}
		stats, _ := f.Stat()
		// Gathering the uid, gid, and mode this way is not portable.
		// This will build on Linux but not OS X. The Mode on OS X returns a
		// uint16 but uint32 on Linux. We could change our FileInfo Mode field
		// type to os.FileMode and cast Mode here.
		fi.Uid = stats.Sys().(*syscall.Stat_t).Uid
		fi.Gid = stats.Sys().(*syscall.Stat_t).Gid
		fi.Mode = stats.Sys().(*syscall.Stat_t).Mode
		h := md5.New()
		io.Copy(h, f)
		fi.Md5 = fmt.Sprintf("%x", h.Sum(nil))
		return fi, nil
	} else {
		return fi, errors.New("File not found")
	}
}

// sameConfig reports whether src and dest config files are equal.
// Two config files are equal when they have the same file contents and
// Unix permissions. The owner, group, and mode must match.
// It return false in other cases.
func sameConfig(src, dest string) (error, bool) {
	if !isFileExist(dest) {
		return nil, false
	}
	d, err := fileStat(dest)
	if err != nil {
		return err, false
	}
	s, err := fileStat(src)
	if err != nil {
		return err, false
	}
	if d.Uid != s.Uid || d.Gid != s.Gid || d.Mode != s.Mode || d.Md5 != s.Md5 {
		return nil, false
	}
	return nil, true
}
<|endoftext|>


<commit_before>package common

import (
	"bytes"
	"fmt"
	"github.com/bwmarrin/discordgo"
	"github.com/dmportella/qilbot/bot"
	"github.com/dmportella/qilbot/utilities"
)

// NewPlugin creates a new instance of Common Plugin.
func NewPlugin(qilbot *bot.Qilbot) (plugin *Plugin) {
	const (
		Name        = "Qilbot Common plugin"
		Description = "Common plugin for qibot a a place for generic commands."
	)

	plugin = &Plugin{
		bot.Plugin{
			Qilbot:      qilbot,
			Name:        Name,
			Description: Description,
			Commands: []bot.CommandInformation{
				{
					Command:     "plugins",
					Template:    "plugins",
					Description: "Display a list of plugins enabled on qilbot.",
				},
				{
					Command:     "help",
					Template:    "help",
					Description: "Display a list of commands available to qilbot.",
				},
			},
		},
	}

	qilbot.AddPlugin(plugin)
	qilbot.AddHandler(plugin.messageCreate)

	return
}

func (plugin *Plugin) messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
	// Ignore all messages created by the bot itself
	if plugin.Plugin.Qilbot.IsBot(m.Author.ID) {
		return
	}

	matches := utilities.RegexMatchBotCommand(m.Content)

	if len(matches) >= 3 && plugin.Plugin.Qilbot.IsBot(matches[1]) {
		switch matches[2] {
		case "plugins":
			plugin.displayPluginList(s, m)
			break
		case "help":
			plugin.displayHelp(s, m)
			break
		default:
			return
		}
	}
}

func (plugin *Plugin) displayPluginList(s *discordgo.Session, m *discordgo.MessageCreate) {
	var buffer bytes.Buffer
	for _, item := range plugin.Plugin.Qilbot.Plugins {
		buffer.WriteString(item.GetHelpText() + "\n")
	}
	_, _ = s.ChannelMessageSend(m.ChannelID, buffer.String())
}

func (plugin *Plugin) displayHelp(s *discordgo.Session, m *discordgo.MessageCreate) {
	var buffer bytes.Buffer
	buffer.WriteString("List of Commands available to Qilbot.\n")
	for _, item := range plugin.Plugin.Qilbot.Plugins {
		buffer.WriteString(fmt.Sprintf("%s\n", item.GetHelpText()))
		for _, command := range item.GetCommands() {
			buffer.WriteString(fmt.Sprintf("\t**%s** (%s): %s\n", command.Command, command.Template, command.Description))

		}
	}
	_, _ = s.ChannelMessageSend(m.ChannelID, buffer.String())
}
<commit_msg>fixed spelling mistake<commit_after>package common

import (
	"bytes"
	"fmt"
	"github.com/bwmarrin/discordgo"
	"github.com/dmportella/qilbot/bot"
	"github.com/dmportella/qilbot/utilities"
)

// NewPlugin creates a new instance of Common Plugin.
func NewPlugin(qilbot *bot.Qilbot) (plugin *Plugin) {
	const (
		Name        = "Qilbot Common plugin"
		Description = "Common plugin for qibot a place for generic commands."
	)

	plugin = &Plugin{
		bot.Plugin{
			Qilbot:      qilbot,
			Name:        Name,
			Description: Description,
			Commands: []bot.CommandInformation{
				{
					Command:     "plugins",
					Template:    "plugins",
					Description: "Display a list of plugins enabled on qilbot.",
				},
				{
					Command:     "help",
					Template:    "help",
					Description: "Display a list of commands available to qilbot.",
				},
			},
		},
	}

	qilbot.AddPlugin(plugin)
	qilbot.AddHandler(plugin.messageCreate)

	return
}

func (plugin *Plugin) messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
	// Ignore all messages created by the bot itself
	if plugin.Plugin.Qilbot.IsBot(m.Author.ID) {
		return
	}

	matches := utilities.RegexMatchBotCommand(m.Content)

	if len(matches) >= 3 && plugin.Plugin.Qilbot.IsBot(matches[1]) {
		switch matches[2] {
		case "plugins":
			plugin.displayPluginList(s, m)
			break
		case "help":
			plugin.displayHelp(s, m)
			break
		default:
			return
		}
	}
}

func (plugin *Plugin) displayPluginList(s *discordgo.Session, m *discordgo.MessageCreate) {
	var buffer bytes.Buffer
	for _, item := range plugin.Plugin.Qilbot.Plugins {
		buffer.WriteString(item.GetHelpText() + "\n")
	}
	_, _ = s.ChannelMessageSend(m.ChannelID, buffer.String())
}

func (plugin *Plugin) displayHelp(s *discordgo.Session, m *discordgo.MessageCreate) {
	var buffer bytes.Buffer
	buffer.WriteString("List of Commands available to Qilbot.\n")
	for _, item := range plugin.Plugin.Qilbot.Plugins {
		buffer.WriteString(fmt.Sprintf("%s\n", item.GetHelpText()))
		for _, command := range item.GetCommands() {
			buffer.WriteString(fmt.Sprintf("\t**%s** (%s): %s\n", command.Command, command.Template, command.Description))

		}
	}
	_, _ = s.ChannelMessageSend(m.ChannelID, buffer.String())
}
<|endoftext|>


<commit_before>package deploy

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/jobtalk/thor/api"
	"github.com/jobtalk/thor/lib"
	"github.com/jobtalk/thor/lib/setting"
)

var re = regexp.MustCompile(`.*\.json$`)

var flagSet = &flag.FlagSet{}
var cred *credentials.Credentials
var (
	file         *string
	f            *string
	profile      *string
	kmsKeyID     *string
	region       *string
	externalPath *string
	outerVals    *string
)

func init() {
	kmsKeyID = flagSet.String("key_id", "", "Amazon KMS key ID")
	file = flagSet.String("file", "", "target file")
	f = flagSet.String("f", "", "target file")
	profile = flagSet.String("profile", "default", "aws credentials profile name")
	region = flagSet.String("region", "ap-northeast-1", "aws region")
	externalPath = flagSet.String("external_path", "", "external conf path")
	outerVals = flagSet.String("V", "", "outer values")
}

func fileList(root string) ([]string, error) {
	if root == "" {
		return nil, nil
	}
	ret := []string{}
	err := filepath.Walk(root,
		func(path string, info os.FileInfo, err error) error {
			if info.IsDir() {
				return nil
			}

			rel, err := filepath.Rel(root, path)
			if re.MatchString(rel) {
				ret = append(ret, rel)
			}

			return nil
		})

	if err != nil {
		return nil, err
	}

	return ret, nil
}

type deployConfigure struct {
	*setting.Setting
}

func isEncrypted(data []byte) bool {
	var buffer = map[string]interface{}{}
	if err := json.Unmarshal(data, &buffer); err != nil {
		return false
	}
	elem, ok := buffer["cipher"]
	if !ok {
		return false
	}
	str, ok := elem.(string)
	if !ok {
		return false
	}

	return len(str) != 0
}

func decrypt(bin []byte) ([]byte, error) {
	awsConfig := &aws.Config{
		Credentials: cred,
		Region:      region,
	}
	kms := lib.NewKMSFromBinary(bin)
	if kms == nil {
		return nil, errors.New(fmt.Sprintf("%v format is illegal", string(bin)))
	}
	plainText, err := kms.SetKeyID(*kmsKeyID).SetAWSConfig(awsConfig).Decrypt()
	if err != nil {
		return nil, err
	}
	return plainText, nil
}

func readConf(base []byte, externalPathList []string) (*deployConfigure, error) {
	var root = *externalPath
	var ret = &deployConfigure{}
	baseStr := string(base)

	root = strings.TrimSuffix(root, "/")
	for _, externalPath := range externalPathList {
		external, err := ioutil.ReadFile(root + "/" + externalPath)
		if err != nil {
			return nil, err
		}
		if isEncrypted(external) {
			plain, err := decrypt(external)
			if err != nil {
				return nil, err
			}
			external = plain
		}
		baseStr, err = lib.Embedde(baseStr, string(external))
		if err != nil {
			return nil, err
		}
	}
	if err := json.Unmarshal([]byte(baseStr), ret); err != nil {
		return nil, err
	}
	return ret, nil
}

type Deploy struct{}

func (c *Deploy) Run(args []string) int {
	var config = &deployConfigure{}
	if err := flagSet.Parse(args); err != nil {
		log.Fatalln(err)
	}
	if *file == "" {
		file = f
	}
	cred = credentials.NewSharedCredentials("", *profile)
	awsConfig := &aws.Config{
		Credentials: cred,
		Region:      region,
	}

	externalList, err := fileList(*externalPath)
	if err != nil {
		log.Fatalln(err)
	}
	baseConfBinary, err := ioutil.ReadFile(*file)
	if err != nil {
		log.Fatal(err)
	}
	baseStr, err := lib.Embedde(string(baseConfBinary), *outerVals)
	if err != nil {
		log.Fatal(err)
	}
	baseConfBinary = []byte(baseStr)
	if externalList != nil {
		c, err := readConf(baseConfBinary, externalList)
		if err != nil {
			log.Fatalln(err)
		}
		config = c
	} else {
		bin, err := ioutil.ReadFile(*file)
		if err != nil {
			log.Fatalln(err)
		}
		if err := json.Unmarshal(bin, config); err != nil {
			log.Fatalln(err)
		}
	}

	result, err := api.Deploy(awsConfig, config.Setting)
	if err != nil {
		log.Fatalln(err)
	}
	fmt.Println(result)
	return 0
}

func (c *Deploy) Synopsis() string {
	synopsis := ""
	synopsis += "usage: thor deploy [options ...]\n"
	synopsis += "options:\n"
	synopsis += "    -f thor_setting.json\n"
	synopsis += "\n"
	synopsis += "    -profile=${aws profile name}\n"
	synopsis += "        -profile option is arbitrary parameter.\n"
	synopsis += "    -region\n"
	synopsis += "        aws region\n"
	synopsis += "    -external_path\n"
	synopsis += "        setting external path file\n"
	synopsis += "    -V\n"
	synopsis += "        setting outer values\n"
	synopsis += "===================================================\n"

	return synopsis
}

func (c *Deploy) Help() string {
	return c.Synopsis()
}
<commit_msg>コマンドライン引数から値を取得する時空だった場合処理しない<commit_after>package deploy

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/jobtalk/thor/api"
	"github.com/jobtalk/thor/lib"
	"github.com/jobtalk/thor/lib/setting"
)

var re = regexp.MustCompile(`.*\.json$`)

var flagSet = &flag.FlagSet{}
var cred *credentials.Credentials
var (
	file         *string
	f            *string
	profile      *string
	kmsKeyID     *string
	region       *string
	externalPath *string
	outerVals    *string
)

func init() {
	kmsKeyID = flagSet.String("key_id", "", "Amazon KMS key ID")
	file = flagSet.String("file", "", "target file")
	f = flagSet.String("f", "", "target file")
	profile = flagSet.String("profile", "default", "aws credentials profile name")
	region = flagSet.String("region", "ap-northeast-1", "aws region")
	externalPath = flagSet.String("external_path", "", "external conf path")
	outerVals = flagSet.String("V", "", "outer values")
}

func fileList(root string) ([]string, error) {
	if root == "" {
		return nil, nil
	}
	ret := []string{}
	err := filepath.Walk(root,
		func(path string, info os.FileInfo, err error) error {
			if info.IsDir() {
				return nil
			}

			rel, err := filepath.Rel(root, path)
			if re.MatchString(rel) {
				ret = append(ret, rel)
			}

			return nil
		})

	if err != nil {
		return nil, err
	}

	return ret, nil
}

type deployConfigure struct {
	*setting.Setting
}

func isEncrypted(data []byte) bool {
	var buffer = map[string]interface{}{}
	if err := json.Unmarshal(data, &buffer); err != nil {
		return false
	}
	elem, ok := buffer["cipher"]
	if !ok {
		return false
	}
	str, ok := elem.(string)
	if !ok {
		return false
	}

	return len(str) != 0
}

func decrypt(bin []byte) ([]byte, error) {
	awsConfig := &aws.Config{
		Credentials: cred,
		Region:      region,
	}
	kms := lib.NewKMSFromBinary(bin)
	if kms == nil {
		return nil, errors.New(fmt.Sprintf("%v format is illegal", string(bin)))
	}
	plainText, err := kms.SetKeyID(*kmsKeyID).SetAWSConfig(awsConfig).Decrypt()
	if err != nil {
		return nil, err
	}
	return plainText, nil
}

func readConf(base []byte, externalPathList []string) (*deployConfigure, error) {
	var root = *externalPath
	var ret = &deployConfigure{}
	baseStr := string(base)

	root = strings.TrimSuffix(root, "/")
	for _, externalPath := range externalPathList {
		external, err := ioutil.ReadFile(root + "/" + externalPath)
		if err != nil {
			return nil, err
		}
		if isEncrypted(external) {
			plain, err := decrypt(external)
			if err != nil {
				return nil, err
			}
			external = plain
		}
		baseStr, err = lib.Embedde(baseStr, string(external))
		if err != nil {
			return nil, err
		}
	}
	if err := json.Unmarshal([]byte(baseStr), ret); err != nil {
		return nil, err
	}
	return ret, nil
}

type Deploy struct{}

func (c *Deploy) Run(args []string) int {
	var config = &deployConfigure{}
	if err := flagSet.Parse(args); err != nil {
		log.Fatalln(err)
	}
	if *file == "" {
		file = f
	}
	cred = credentials.NewSharedCredentials("", *profile)
	awsConfig := &aws.Config{
		Credentials: cred,
		Region:      region,
	}

	externalList, err := fileList(*externalPath)
	if err != nil {
		log.Fatalln(err)
	}
	baseConfBinary, err := ioutil.ReadFile(*file)
	if err != nil {
		log.Fatal(err)
	}

	if *outerVals != "" {
		baseStr, err := lib.Embedde(string(baseConfBinary), *outerVals)
		if err != nil {
			log.Fatal(err)
		}
		baseConfBinary = []byte(baseStr)
	}

	if externalList != nil {
		c, err := readConf(baseConfBinary, externalList)
		if err != nil {
			log.Fatalln(err)
		}
		config = c
	} else {
		bin, err := ioutil.ReadFile(*file)
		if err != nil {
			log.Fatalln(err)
		}
		if err := json.Unmarshal(bin, config); err != nil {
			log.Fatalln(err)
		}
	}

	result, err := api.Deploy(awsConfig, config.Setting)
	if err != nil {
		log.Fatalln(err)
	}
	fmt.Println(result)
	return 0
}

func (c *Deploy) Synopsis() string {
	synopsis := ""
	synopsis += "usage: thor deploy [options ...]\n"
	synopsis += "options:\n"
	synopsis += "    -f thor_setting.json\n"
	synopsis += "\n"
	synopsis += "    -profile=${aws profile name}\n"
	synopsis += "        -profile option is arbitrary parameter.\n"
	synopsis += "    -region\n"
	synopsis += "        aws region\n"
	synopsis += "    -external_path\n"
	synopsis += "        setting external path file\n"
	synopsis += "    -V\n"
	synopsis += "        setting outer values\n"
	synopsis += "===================================================\n"

	return synopsis
}

func (c *Deploy) Help() string {
	return c.Synopsis()
}
<|endoftext|>