<commit_before>/*
 * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"errors"
	"fmt"
	"sort"
	"sync"
	"time"

	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/bpool"
	"github.com/minio/minio/pkg/color"
	"github.com/minio/minio/pkg/dsync"
	"github.com/minio/minio/pkg/madmin"
	"github.com/minio/minio/pkg/sync/errgroup"
)

// OfflineDisk represents an unavailable disk.
var OfflineDisk StorageAPI // zero value is nil

// partialOperation is a successful upload/delete of an object
// but not written in all disks (having quorum)
type partialOperation struct {
	bucket    string
	object    string
	versionID string
	failedSet int
}

// erasureObjects - Implements ER object layer.
type erasureObjects struct {
	GatewayUnsupported

	setDriveCount      int
	defaultParityCount int

	setNumber int

	// getDisks returns list of storageAPIs.
	getDisks func() []StorageAPI

	// getLockers returns list of remote and local lockers.
	getLockers func() ([]dsync.NetLocker, string)

	// getEndpoints returns list of endpoint strings belonging this set.
	// some may be local and some remote.
	getEndpoints func() []string

	// Locker mutex map.
	nsMutex *nsLockMap

	// Byte pools used for temporary i/o buffers.
	bp *bpool.BytePoolCap

	mrfOpCh chan partialOperation
}

// NewNSLock - initialize a new namespace RWLocker instance.
func (er erasureObjects) NewNSLock(bucket string, objects ...string) RWLocker {
	return er.nsMutex.NewNSLock(er.getLockers, bucket, objects...)
}

// Shutdown function for object storage interface.
func (er erasureObjects) Shutdown(ctx context.Context) error {
	// Add any object layer shutdown activities here.
	closeStorageDisks(er.getDisks())
	return nil
}

// byDiskTotal is a collection satisfying sort.Interface.
type byDiskTotal []madmin.Disk

func (d byDiskTotal) Len() int      { return len(d) }
func (d byDiskTotal) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d byDiskTotal) Less(i, j int) bool {
	return d[i].TotalSpace < d[j].TotalSpace
}

func diskErrToDriveState(err error) (state string) {
	state = madmin.DriveStateUnknown
	switch {
	case errors.Is(err, errDiskNotFound):
		state = madmin.DriveStateOffline
	case errors.Is(err, errCorruptedFormat):
		state = madmin.DriveStateCorrupt
	case errors.Is(err, errUnformattedDisk):
		state = madmin.DriveStateUnformatted
	case errors.Is(err, errDiskAccessDenied):
		state = madmin.DriveStatePermission
	case errors.Is(err, errFaultyDisk):
		state = madmin.DriveStateFaulty
	case err == nil:
		state = madmin.DriveStateOk
	}
	return
}

func getOnlineOfflineDisksStats(disksInfo []madmin.Disk) (onlineDisks, offlineDisks madmin.BackendDisks) {
	onlineDisks = make(madmin.BackendDisks)
	offlineDisks = make(madmin.BackendDisks)

	for _, disk := range disksInfo {
		ep := disk.Endpoint
		if _, ok := offlineDisks[ep]; !ok {
			offlineDisks[ep] = 0
		}
		if _, ok := onlineDisks[ep]; !ok {
			onlineDisks[ep] = 0
		}
	}

	// Wait for the routines.
	for _, disk := range disksInfo {
		ep := disk.Endpoint
		state := disk.State
		if state != madmin.DriveStateOk && state != madmin.DriveStateUnformatted {
			offlineDisks[ep]++
			continue
		}
		onlineDisks[ep]++
	}

	rootDiskCount := 0
	for _, di := range disksInfo {
		if di.RootDisk {
			rootDiskCount++
		}
	}

	// Count offline disks as well to ensure consistent
	// reportability of offline drives on local setups.
	if len(disksInfo) == (rootDiskCount + offlineDisks.Sum()) {
		// Success.
		return onlineDisks, offlineDisks
	}

	// Root disk should be considered offline
	for i := range disksInfo {
		ep := disksInfo[i].Endpoint
		if disksInfo[i].RootDisk {
			offlineDisks[ep]++
			onlineDisks[ep]--
		}
	}

	return onlineDisks, offlineDisks
}

// getDisksInfo - fetch disks info across all other storage API.
func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Disk, errs []error) {
	disksInfo = make([]madmin.Disk, len(disks))

	g := errgroup.WithNErrs(len(disks))
	for index := range disks {
		index := index
		g.Go(func() error {
			if disks[index] == OfflineDisk {
				logger.LogIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]))
				disksInfo[index] = madmin.Disk{
					State:    diskErrToDriveState(errDiskNotFound),
					Endpoint: endpoints[index],
				}
				// Storage disk is empty, perhaps ignored disk or not available.
				return errDiskNotFound
			}
			info, err := disks[index].DiskInfo(context.TODO())
			di := madmin.Disk{
				Endpoint:       endpoints[index],
				DrivePath:      info.MountPath,
				TotalSpace:     info.Total,
				UsedSpace:      info.Used,
				AvailableSpace: info.Free,
				UUID:           info.ID,
				RootDisk:       info.RootDisk,
				Healing:        info.Healing,
				State:          diskErrToDriveState(err),
			}
			if info.Total > 0 {
				di.Utilization = float64(info.Used / info.Total * 100)
			}
			disksInfo[index] = di
			return err
		}, index)
	}

	return disksInfo, g.Wait()
}

// Get an aggregated storage info across all disks.
func getStorageInfo(disks []StorageAPI, endpoints []string) (StorageInfo, []error) {
	disksInfo, errs := getDisksInfo(disks, endpoints)

	// Sort so that the first element is the smallest.
	sort.Sort(byDiskTotal(disksInfo))

	storageInfo := StorageInfo{
		Disks: disksInfo,
	}

	storageInfo.Backend.Type = BackendErasure
	return storageInfo, errs
}

// StorageInfo - returns underlying storage statistics.
func (er erasureObjects) StorageInfo(ctx context.Context) (StorageInfo, []error) {
	disks := er.getDisks()
	endpoints := er.getEndpoints()
	return getStorageInfo(disks, endpoints)
}

func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, healing bool) {
	var wg sync.WaitGroup
	disks := er.getDisks()
	infos := make([]DiskInfo, len(disks))
	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()

			disk := disks[i-1]

			if disk == nil {
				infos[i-1].Error = "nil disk"
				return
			}

			di, err := disk.DiskInfo(context.Background())
			if err != nil {
				// - Do not consume disks which are not reachable
				//   unformatted or simply not accessible for some reason.
				//
				//
				// - Future: skip busy disks
				infos[i-1].Error = err.Error()
				return
			}

			infos[i-1] = di
		}()
	}
	wg.Wait()

	for i, info := range infos {
		// Check if one of the drives in the set is being healed.
		// this information is used by crawler to skip healing
		// this erasure set while it calculates the usage.
		if info.Healing || info.Error != "" {
			healing = true
			continue
		}
		newDisks = append(newDisks, disks[i])
	}

	return newDisks, healing
}

// CrawlAndGetDataUsage will start crawling buckets and send updated totals as they are traversed.
// Updates are sent on a regular basis and the caller *must* consume them.
func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, updates chan<- dataUsageCache) error {
	if len(buckets) == 0 {
		logger.Info(color.Green("data-crawl:") + " No buckets found, skipping crawl")
		return nil
	}

	// Collect disks we can use.
	disks, healing := er.getOnlineDisksWithHealing()
	if len(disks) == 0 {
		logger.Info(color.Green("data-crawl:") + " all disks are offline or being healed, skipping crawl")
		return nil
	}

	// Collect disks for healing.
	allDisks := er.getDisks()
	allDiskIDs := make([]string, 0, len(allDisks))
	for _, disk := range allDisks {
		if disk == OfflineDisk {
			// its possible that disk is OfflineDisk
			continue
		}
		id, _ := disk.GetDiskID()
		if id == "" {
			// its possible that disk is unformatted
			// or just went offline
			continue
		}
		allDiskIDs = append(allDiskIDs, id)
	}

	// Load bucket totals
	oldCache := dataUsageCache{}
	if err := oldCache.load(ctx, er, dataUsageCacheName); err != nil {
		return err
	}

	// New cache..
	cache := dataUsageCache{
		Info: dataUsageCacheInfo{
			Name:      dataUsageRoot,
			NextCycle: oldCache.Info.NextCycle,
		},
		Cache: make(map[string]dataUsageEntry, len(oldCache.Cache)),
	}
	bloom := bf.bytes()

	// Put all buckets into channel.
	bucketCh := make(chan BucketInfo, len(buckets))
	// Add new buckets first
	for _, b := range buckets {
		if oldCache.find(b.Name) == nil {
			bucketCh <- b
		}
	}

	// Add existing buckets.
	for _, b := range buckets {
		e := oldCache.find(b.Name)
		if e != nil {
			cache.replace(b.Name, dataUsageRoot, *e)
			bucketCh <- b
		}
	}

	close(bucketCh)
	bucketResults := make(chan dataUsageEntryInfo, len(disks))

	// Start async collector/saver.
	// This goroutine owns the cache.
	var saverWg sync.WaitGroup
	saverWg.Add(1)
	go func() {
		const updateTime = 30 * time.Second
		t := time.NewTicker(updateTime)
		defer t.Stop()
		defer saverWg.Done()
		var lastSave time.Time

		for {
			select {
			case <-ctx.Done():
				// Return without saving.
				return
			case <-t.C:
				if cache.Info.LastUpdate.Equal(lastSave) {
					continue
				}
				logger.LogIf(ctx, cache.save(ctx, er, dataUsageCacheName))
				updates <- cache.clone()
				lastSave = cache.Info.LastUpdate
			case v, ok := <-bucketResults:
				if !ok {
					// Save final state...
					cache.Info.NextCycle++
					cache.Info.LastUpdate = time.Now()
					logger.LogIf(ctx, cache.save(ctx, er, dataUsageCacheName))
					updates <- cache
					return
				}
				cache.replace(v.Name, v.Parent, v.Entry)
				cache.Info.LastUpdate = time.Now()
			}
		}
	}()

	// Start one crawler per disk
	var wg sync.WaitGroup
	wg.Add(len(disks))
	for i := range disks {
		go func(i int) {
			defer wg.Done()
			disk := disks[i]

			for bucket := range bucketCh {
				select {
				case <-ctx.Done():
					return
				default:
				}

				// Load cache for bucket
				cacheName := pathJoin(bucket.Name, dataUsageCacheName)
				cache := dataUsageCache{}
				logger.LogIf(ctx, cache.load(ctx, er, cacheName))
				if cache.Info.Name == "" {
					cache.Info.Name = bucket.Name
				}
				cache.Info.BloomFilter = bloom
				cache.Info.SkipHealing = healing
				cache.Disks = allDiskIDs
				if cache.Info.Name != bucket.Name {
					logger.LogIf(ctx, fmt.Errorf("cache name mismatch: %s != %s", cache.Info.Name, bucket.Name))
					cache.Info = dataUsageCacheInfo{
						Name:       bucket.Name,
						LastUpdate: time.Time{},
						NextCycle:  0,
					}
				}

				// Calc usage
				before := cache.Info.LastUpdate
				var err error
				cache, err = disk.CrawlAndGetDataUsage(ctx, cache)
				cache.Info.BloomFilter = nil
				if err != nil {
					logger.LogIf(ctx, err)
					if cache.Info.LastUpdate.After(before) {
						logger.LogIf(ctx, cache.save(ctx, er, cacheName))
					}
					continue
				}

				var root dataUsageEntry
				if r := cache.root(); r != nil {
					root = cache.flatten(*r)
				}
				bucketResults <- dataUsageEntryInfo{
					Name:   cache.Info.Name,
					Parent: dataUsageRoot,
					Entry:  root,
				}
				// Save cache
				logger.LogIf(ctx, cache.save(ctx, er, cacheName))
			}
		}(i)
	}
	wg.Wait()
	close(bucketResults)
	saverWg.Wait()

	return nil
}
<commit_msg>Reduce redundant crawler logging (#11448)<commit_after>/*
 * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"errors"
	"fmt"
	"sort"
	"sync"
	"time"

	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/bpool"
	"github.com/minio/minio/pkg/color"
	"github.com/minio/minio/pkg/dsync"
	"github.com/minio/minio/pkg/madmin"
	"github.com/minio/minio/pkg/sync/errgroup"
)

// OfflineDisk represents an unavailable disk.
var OfflineDisk StorageAPI // zero value is nil

// partialOperation is a successful upload/delete of an object
// but not written in all disks (having quorum)
type partialOperation struct {
	bucket    string
	object    string
	versionID string
	failedSet int
}

// erasureObjects - Implements ER object layer.
type erasureObjects struct {
	GatewayUnsupported

	setDriveCount      int
	defaultParityCount int

	setNumber int

	// getDisks returns list of storageAPIs.
	getDisks func() []StorageAPI

	// getLockers returns list of remote and local lockers.
	getLockers func() ([]dsync.NetLocker, string)

	// getEndpoints returns list of endpoint strings belonging this set.
	// some may be local and some remote.
	getEndpoints func() []string

	// Locker mutex map.
	nsMutex *nsLockMap

	// Byte pools used for temporary i/o buffers.
	bp *bpool.BytePoolCap

	mrfOpCh chan partialOperation
}

// NewNSLock - initialize a new namespace RWLocker instance.
func (er erasureObjects) NewNSLock(bucket string, objects ...string) RWLocker {
	return er.nsMutex.NewNSLock(er.getLockers, bucket, objects...)
}

// Shutdown function for object storage interface.
func (er erasureObjects) Shutdown(ctx context.Context) error {
	// Add any object layer shutdown activities here.
	closeStorageDisks(er.getDisks())
	return nil
}

// byDiskTotal is a collection satisfying sort.Interface.
type byDiskTotal []madmin.Disk

func (d byDiskTotal) Len() int      { return len(d) }
func (d byDiskTotal) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d byDiskTotal) Less(i, j int) bool {
	return d[i].TotalSpace < d[j].TotalSpace
}

func diskErrToDriveState(err error) (state string) {
	state = madmin.DriveStateUnknown
	switch {
	case errors.Is(err, errDiskNotFound):
		state = madmin.DriveStateOffline
	case errors.Is(err, errCorruptedFormat):
		state = madmin.DriveStateCorrupt
	case errors.Is(err, errUnformattedDisk):
		state = madmin.DriveStateUnformatted
	case errors.Is(err, errDiskAccessDenied):
		state = madmin.DriveStatePermission
	case errors.Is(err, errFaultyDisk):
		state = madmin.DriveStateFaulty
	case err == nil:
		state = madmin.DriveStateOk
	}
	return
}

func getOnlineOfflineDisksStats(disksInfo []madmin.Disk) (onlineDisks, offlineDisks madmin.BackendDisks) {
	onlineDisks = make(madmin.BackendDisks)
	offlineDisks = make(madmin.BackendDisks)

	for _, disk := range disksInfo {
		ep := disk.Endpoint
		if _, ok := offlineDisks[ep]; !ok {
			offlineDisks[ep] = 0
		}
		if _, ok := onlineDisks[ep]; !ok {
			onlineDisks[ep] = 0
		}
	}

	// Wait for the routines.
	for _, disk := range disksInfo {
		ep := disk.Endpoint
		state := disk.State
		if state != madmin.DriveStateOk && state != madmin.DriveStateUnformatted {
			offlineDisks[ep]++
			continue
		}
		onlineDisks[ep]++
	}

	rootDiskCount := 0
	for _, di := range disksInfo {
		if di.RootDisk {
			rootDiskCount++
		}
	}

	// Count offline disks as well to ensure consistent
	// reportability of offline drives on local setups.
	if len(disksInfo) == (rootDiskCount + offlineDisks.Sum()) {
		// Success.
		return onlineDisks, offlineDisks
	}

	// Root disk should be considered offline
	for i := range disksInfo {
		ep := disksInfo[i].Endpoint
		if disksInfo[i].RootDisk {
			offlineDisks[ep]++
			onlineDisks[ep]--
		}
	}

	return onlineDisks, offlineDisks
}

// getDisksInfo - fetch disks info across all other storage API.
func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Disk, errs []error) {
	disksInfo = make([]madmin.Disk, len(disks))

	g := errgroup.WithNErrs(len(disks))
	for index := range disks {
		index := index
		g.Go(func() error {
			if disks[index] == OfflineDisk {
				logger.LogIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]))
				disksInfo[index] = madmin.Disk{
					State:    diskErrToDriveState(errDiskNotFound),
					Endpoint: endpoints[index],
				}
				// Storage disk is empty, perhaps ignored disk or not available.
				return errDiskNotFound
			}
			info, err := disks[index].DiskInfo(context.TODO())
			di := madmin.Disk{
				Endpoint:       endpoints[index],
				DrivePath:      info.MountPath,
				TotalSpace:     info.Total,
				UsedSpace:      info.Used,
				AvailableSpace: info.Free,
				UUID:           info.ID,
				RootDisk:       info.RootDisk,
				Healing:        info.Healing,
				State:          diskErrToDriveState(err),
			}
			if info.Total > 0 {
				di.Utilization = float64(info.Used / info.Total * 100)
			}
			disksInfo[index] = di
			return err
		}, index)
	}

	return disksInfo, g.Wait()
}

// Get an aggregated storage info across all disks.
func getStorageInfo(disks []StorageAPI, endpoints []string) (StorageInfo, []error) {
	disksInfo, errs := getDisksInfo(disks, endpoints)

	// Sort so that the first element is the smallest.
	sort.Sort(byDiskTotal(disksInfo))

	storageInfo := StorageInfo{
		Disks: disksInfo,
	}

	storageInfo.Backend.Type = BackendErasure
	return storageInfo, errs
}

// StorageInfo - returns underlying storage statistics.
func (er erasureObjects) StorageInfo(ctx context.Context) (StorageInfo, []error) {
	disks := er.getDisks()
	endpoints := er.getEndpoints()
	return getStorageInfo(disks, endpoints)
}

func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, healing bool) {
	var wg sync.WaitGroup
	disks := er.getDisks()
	infos := make([]DiskInfo, len(disks))
	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()

			disk := disks[i-1]

			if disk == nil {
				infos[i-1].Error = "nil disk"
				return
			}

			di, err := disk.DiskInfo(context.Background())
			if err != nil {
				// - Do not consume disks which are not reachable
				//   unformatted or simply not accessible for some reason.
				//
				//
				// - Future: skip busy disks
				infos[i-1].Error = err.Error()
				return
			}

			infos[i-1] = di
		}()
	}
	wg.Wait()

	for i, info := range infos {
		// Check if one of the drives in the set is being healed.
		// this information is used by crawler to skip healing
		// this erasure set while it calculates the usage.
		if info.Healing || info.Error != "" {
			healing = true
			continue
		}
		newDisks = append(newDisks, disks[i])
	}

	return newDisks, healing
}

// CrawlAndGetDataUsage will start crawling buckets and send updated totals as they are traversed.
// Updates are sent on a regular basis and the caller *must* consume them.
func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, updates chan<- dataUsageCache) error {
	if len(buckets) == 0 {
		return nil
	}

	// Collect disks we can use.
	disks, healing := er.getOnlineDisksWithHealing()
	if len(disks) == 0 {
		logger.Info(color.Green("data-crawl:") + " all disks are offline or being healed, skipping crawl")
		return nil
	}

	// Collect disks for healing.
	allDisks := er.getDisks()
	allDiskIDs := make([]string, 0, len(allDisks))
	for _, disk := range allDisks {
		if disk == OfflineDisk {
			// its possible that disk is OfflineDisk
			continue
		}
		id, _ := disk.GetDiskID()
		if id == "" {
			// its possible that disk is unformatted
			// or just went offline
			continue
		}
		allDiskIDs = append(allDiskIDs, id)
	}

	// Load bucket totals
	oldCache := dataUsageCache{}
	if err := oldCache.load(ctx, er, dataUsageCacheName); err != nil {
		return err
	}

	// New cache..
	cache := dataUsageCache{
		Info: dataUsageCacheInfo{
			Name:      dataUsageRoot,
			NextCycle: oldCache.Info.NextCycle,
		},
		Cache: make(map[string]dataUsageEntry, len(oldCache.Cache)),
	}
	bloom := bf.bytes()

	// Put all buckets into channel.
	bucketCh := make(chan BucketInfo, len(buckets))
	// Add new buckets first
	for _, b := range buckets {
		if oldCache.find(b.Name) == nil {
			bucketCh <- b
		}
	}

	// Add existing buckets.
	for _, b := range buckets {
		e := oldCache.find(b.Name)
		if e != nil {
			cache.replace(b.Name, dataUsageRoot, *e)
			bucketCh <- b
		}
	}

	close(bucketCh)
	bucketResults := make(chan dataUsageEntryInfo, len(disks))

	// Start async collector/saver.
	// This goroutine owns the cache.
	var saverWg sync.WaitGroup
	saverWg.Add(1)
	go func() {
		const updateTime = 30 * time.Second
		t := time.NewTicker(updateTime)
		defer t.Stop()
		defer saverWg.Done()
		var lastSave time.Time

		for {
			select {
			case <-ctx.Done():
				// Return without saving.
				return
			case <-t.C:
				if cache.Info.LastUpdate.Equal(lastSave) {
					continue
				}
				logger.LogIf(ctx, cache.save(ctx, er, dataUsageCacheName))
				updates <- cache.clone()
				lastSave = cache.Info.LastUpdate
			case v, ok := <-bucketResults:
				if !ok {
					// Save final state...
					cache.Info.NextCycle++
					cache.Info.LastUpdate = time.Now()
					logger.LogIf(ctx, cache.save(ctx, er, dataUsageCacheName))
					updates <- cache
					return
				}
				cache.replace(v.Name, v.Parent, v.Entry)
				cache.Info.LastUpdate = time.Now()
			}
		}
	}()

	// Start one crawler per disk
	var wg sync.WaitGroup
	wg.Add(len(disks))
	for i := range disks {
		go func(i int) {
			defer wg.Done()
			disk := disks[i]

			for bucket := range bucketCh {
				select {
				case <-ctx.Done():
					return
				default:
				}

				// Load cache for bucket
				cacheName := pathJoin(bucket.Name, dataUsageCacheName)
				cache := dataUsageCache{}
				logger.LogIf(ctx, cache.load(ctx, er, cacheName))
				if cache.Info.Name == "" {
					cache.Info.Name = bucket.Name
				}
				cache.Info.BloomFilter = bloom
				cache.Info.SkipHealing = healing
				cache.Disks = allDiskIDs
				if cache.Info.Name != bucket.Name {
					logger.LogIf(ctx, fmt.Errorf("cache name mismatch: %s != %s", cache.Info.Name, bucket.Name))
					cache.Info = dataUsageCacheInfo{
						Name:       bucket.Name,
						LastUpdate: time.Time{},
						NextCycle:  0,
					}
				}

				// Calc usage
				before := cache.Info.LastUpdate
				var err error
				cache, err = disk.CrawlAndGetDataUsage(ctx, cache)
				cache.Info.BloomFilter = nil
				if err != nil {
					logger.LogIf(ctx, err)
					if cache.Info.LastUpdate.After(before) {
						logger.LogIf(ctx, cache.save(ctx, er, cacheName))
					}
					continue
				}

				var root dataUsageEntry
				if r := cache.root(); r != nil {
					root = cache.flatten(*r)
				}
				bucketResults <- dataUsageEntryInfo{
					Name:   cache.Info.Name,
					Parent: dataUsageRoot,
					Entry:  root,
				}
				// Save cache
				logger.LogIf(ctx, cache.save(ctx, er, cacheName))
			}
		}(i)
	}
	wg.Wait()
	close(bucketResults)
	saverWg.Wait()

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"sort"
	"strings"

	"github.com/stts-se/rbg2p"

	"github.com/sergi/go-diff/diffmatchpatch"
)

var l = log.New(os.Stderr, "", 0)

func print(input string, orth string, transes []string) {
	fmt.Printf("%s\t%s\n", input, strings.Join(transes, "  #  "))
}

type transResult struct {
	orth    string
	transes []string
	result  bool
}

func transcribe(ruleSet rbg2p.RuleSet, orth string) transResult {
	transes, err := ruleSet.Apply(orth)
	if err != nil {
		l.Printf("Couldn't transcribe '%s' : %s", orth, err)
		return transResult{orth: orth, transes: transes, result: false}
	}
	return transResult{orth: orth, transes: transes, result: true}
}

var removeStressAndBoundaries = regexp.MustCompile("[.\"%!~] *")

func cleanTransForDiff(t string) string {
	var res = t
	res = removeStressAndBoundaries.ReplaceAllString(res, "")
	//res = strings.Replace(res, "'", "", -1)
	return res
}

func cleanTransForIJDiff(t string) string {
	var res = t
	res = strings.Replace(res, " i ", " j ", -1)
	return res
}

func compareForDiff(old []string, new []string) (string, bool) {
	for i, s := range old {
		old[i] = cleanTransForDiff(s)
	}
	for i, s := range new {
		new[i] = cleanTransForDiff(s)
	}
	// var oldIJ = []string{}
	// var newIJ = []string{}
	// for _, s := range old {
	// 	oldIJ = append(oldIJ, cleanTransForIJDiff(s))
	// }
	// for _, s := range new {
	// 	newIJ = append(newIJ, cleanTransForIJDiff(s))
	// }
	if reflect.DeepEqual(old, new) {
		return "ALL EQ", true
	} else if old[0] == new[0] {
		return "#1 EQ", false
		// } else if reflect.DeepEqual(oldIJ, newIJ) {
		// 	return "ALL EQ IJ", false
		// } else if oldIJ[0] == newIJ[0] {
		// 	return "#1 EQ IJ", false
	} else {
		return "DIFF", false
	}
}

func main() {

	var f = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
	var debug = f.Bool("debug", false, "print extra debug info (default: false)")
	var force = f.Bool("force", false, "print transcriptions even if errors are found (default: false)")
	var column = f.Int("column", 0, "only convert specified column (default: first field)")
	var quiet = f.Bool("quiet", false, "inhibit warnings (default: false)")
	var test = f.Bool("test", false, "test g2p against input file; orth <tab> trans (default: false)")
	var ssFile = f.String("symbolset", "", "use specified symbol set file for validating the symbols in the g2p rule set (default: none; overrides the g2p rule file's symbolset, if any)")
	var help = f.Bool("help", false, "print help message")

	var usage = `go run g2p.go <FLAGS> <G2P RULE FILE> <WORDS (FILES OR LIST OF WORDS)> (optional)

FLAGS:
   -force      bool    print transcriptions even if errors are found (default: false)
   -debug      bool    print extra debug info (default: false)
   -column     string  only convert specified column (default: first field)
   -quiet      bool    inhibit warnings (default: false)
   -test       bool    test g2p against input file; orth <tab> trans (default: false)
   -symbolset  string  use specified symbol set file for validating the symbols in the g2p rule set (default: none)
   -help       bool    print help message`

	f.Usage = func() {
		l.Printf(usage)
	}

	var args = os.Args
	if strings.HasSuffix(args[0], "g2p") {
		args = args[1:] // remove first argument if it's the program name
	}
	err := f.Parse(args)
	if err != nil {
		os.Exit(1)
	}

	args = f.Args()

	if *help {
		l.Println(usage)
		os.Exit(1)
	}

	if len(args) < 1 {
		l.Println(usage)
		os.Exit(1)
	}

	rbg2p.Debug = *debug

	g2pFile := args[0]
	ruleSet, err := rbg2p.LoadFile(g2pFile)
	if err != nil {
		l.Printf("couldn't load rule file %s : %s", g2pFile, err)
		os.Exit(1)
	}

	if *ssFile != "" {
		phonemeSet, err := rbg2p.LoadPhonemeSetFile(*ssFile, ruleSet.PhonemeDelimiter)
		if err != nil {
			l.Printf("couldn't load symbol set : %s", err)
			os.Exit(1)
		}
		ruleSet.PhonemeSet = phonemeSet
	}

	haltingError := false
	result := ruleSet.Test()
	for _, e := range result.Errors {
		l.Printf("ERROR: %v\n", e)
	}
	l.Printf("%d ERROR(S) FOR %s\n", len(result.Errors), g2pFile)
	if !*quiet {
		for _, e := range result.Warnings {
			l.Printf("WARNING: %v\n", e)
		}
	}
	l.Printf("%d WARNING(S) FOR %s\n", len(result.Warnings), g2pFile)
	if len(result.Errors) > 0 {
		haltingError = true
	}
	if len(result.FailedTests) > 0 {
		for _, e := range result.FailedTests {
			l.Printf("FAILED TEST: %v\n", e)
		}
		l.Printf("%d OF %d TESTS FAILED FOR %s\n", len(result.FailedTests), len(ruleSet.Tests), g2pFile)
		haltingError = true
	} else {
		l.Printf("ALL %d TESTS PASSED FOR %s\n", len(ruleSet.Tests), g2pFile)
	}

	if haltingError && !*force {
		os.Exit(1)
	}

	nTotal := 0
	nErrs := 0
	nTrans := 0
	nTests := 0
	testRes := make(map[string]int)
	if *test {
		fmt.Println("ORTH\tNEW TRANSES\tOLD TRANSES\tDIFFTAG\t(DIFF)?")
	}
	var processString = func(s string) {
		nTotal = nTotal + 1
		fs := strings.Split(s, "\t")
		o := fs[*column]
		res := transcribe(ruleSet, o)
		if res.result || *force {
			nTrans = nTrans + 1
			if *test {
				refTranses := fs[(*column + 1):]
				nTests++
				info, _ := compareForDiff(res.transes, refTranses)
				testRes[info]++
				outFs := []string{res.orth, strings.Join(res.transes, " # "), strings.Join(refTranses, "#"), info}
				if info == "DIFF" {
					dmp := diffmatchpatch.New()
					diffs := dmp.DiffMain(outFs[1], outFs[2], false)
					diffsOnly := []diffmatchpatch.Diff{}
					diffsOnlyText := []string{}
					for _, d := range diffs {
						if d.Type != diffmatchpatch.DiffEqual {
							diffsOnly = append(diffsOnly, d)
							diffsOnlyText = append(diffsOnlyText, d.Text)
						}
					}
					outFs = append(outFs, dmp.DiffPrettyText(diffs))
					outFs = append(outFs, fmt.Sprintf("%v", diffsOnly))
					outFs = append(outFs, strings.Join(diffsOnlyText, "|"))
				}

				fmt.Println(strings.Join(outFs, "\t"))
			} else {
				print(s, res.orth, res.transes)
			}
		}
		if !res.result {
			nErrs = nErrs + 1
		}
	}

	if len(args) > 1 {
		for i := 1; i < len(args); i++ {
			s := args[i]
			if _, err := os.Stat(s); os.IsNotExist(err) {
				processString(s)
				// nTotal = nTotal + 1
				// res := transcribe(ruleSet, s)
				// if res.result || *force {
				// 	nTrans = nTrans + 1
				// 	fmt.Printf("%s\t%s\n", s, strings.Join(res.transes, "\t"))
				// }
				// if !res.result {
				// 	nErrs = nErrs + 1
				// }
			} else {
				fh, err := os.Open(filepath.Clean(s))
				if err != nil {
					l.Println(err)
					os.Exit(1)
				}
				defer fh.Close()
				sc := bufio.NewScanner(fh)
				for sc.Scan() {
					if err := sc.Err(); err != nil {
						l.Println(err)
						os.Exit(1)
					}
					line := sc.Text()
					if strings.TrimSpace(line) == "" {
						l.Println("Skipping empty line")
						continue
					}
					if strings.HasPrefix(strings.TrimSpace(line), "#") {
						l.Println("Skipping line " + line)
						continue
					}
					processString(line)
					// 	nTotal = nTotal + 1
					// 	fs := strings.Split(line, "\t")
					// 	o, refTranses := fs[0], fs[1:]
					// 	res := transcribe(ruleSet, o)
					// 	if res.result || *force {
					// 		nTrans = nTrans + 1
					// 		if *test {
					// 			nTests++
					// 			info, _ := compareForDiff(res.transes, refTranses)
					// 			testRes[info]++
					// 			outFs := []string{res.orth, strings.Join(res.transes, " # "), strings.Join(refTranses, "#"), info}
					// 			if info == "DIFF" {
					// 				dmp := diffmatchpatch.New()
					// 				diffs := dmp.DiffMain(outFs[1], outFs[2], false)
					// 				diffsOnly := []diffmatchpatch.Diff{}
					// 				diffsOnlyText := []string{}
					// 				for _, d := range diffs {
					// 					if d.Type != diffmatchpatch.DiffEqual {
					// 						diffsOnly = append(diffsOnly, d)
					// 						diffsOnlyText = append(diffsOnlyText, d.Text)
					// 					}
					// 				}
					// 				outFs = append(outFs, dmp.DiffPrettyText(diffs))
					// 				outFs = append(outFs, fmt.Sprintf("%v", diffsOnly))
					// 				outFs = append(outFs, strings.Join(diffsOnlyText, "|"))
					// 			}

					// 			fmt.Println(strings.Join(outFs, "\t"))
					// 		} else {
					// 			print(res.orth, res.transes)
					// 		}
					// 	}
					// 	if !res.result {
					// 		nErrs = nErrs + 1
					// 	}
				}
			}
		}
	} else {
		fmt.Fprintf(os.Stderr, "Reading input from stdin...\n")
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			line := scanner.Text()
			if strings.TrimSpace(line) == "" {
				l.Println("Skipping empty line")
				continue
			}
			if strings.HasPrefix(strings.TrimSpace(line), "#") {
				l.Println("Skipping line " + line)
				continue
			}
			processString(line)
		}
	}
	l.Printf("%-18s: % 7d", "TOTAL INPUT", nTotal)
	l.Printf("%-18s: % 7d", "ERRORS", nErrs)
	l.Printf("%-18s: % 7d", "TRANSCRIBED", nTrans)
	if *test {
		l.Printf("%-18s: % 7d", "TESTED", nTests)
		var keys []string
		for k := range testRes {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for _, tag := range keys {
			freq := testRes[tag]
			s := " > TEST " + tag
			l.Printf("%-18s: % 7d", s, freq)
		}
	}
}
<commit_msg>g2p.go removing stress is now optional when using the -test switch (-testremovestress)<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"sort"
	"strings"

	"github.com/stts-se/rbg2p"

	"github.com/sergi/go-diff/diffmatchpatch"
)

var l = log.New(os.Stderr, "", 0)

func print(input string, orth string, transes []string) {
	fmt.Printf("%s\t%s\n", input, strings.Join(transes, "  #  "))
}

type transResult struct {
	orth    string
	transes []string
	result  bool
}

func transcribe(ruleSet rbg2p.RuleSet, orth string) transResult {
	transes, err := ruleSet.Apply(orth)
	if err != nil {
		l.Printf("Couldn't transcribe '%s' : %s", orth, err)
		return transResult{orth: orth, transes: transes, result: false}
	}
	return transResult{orth: orth, transes: transes, result: true}
}

var removeBoundariesRE = regexp.MustCompile(`[.!~] *`)
var removeStressRE = regexp.MustCompile(`[%"] *`)
var removeStress *bool

func cleanTransForDiff(t string) string {
	var res = t
	res = removeBoundariesRE.ReplaceAllString(res, "")
	if *removeStress {
		res = removeStressRE.ReplaceAllString(res, "")
	}
	//res = strings.Replace(res, "'", "", -1)
	return res
}

func cleanTransForIJDiff(t string) string {
	var res = t
	res = strings.Replace(res, " i ", " j ", -1)
	return res
}

func compareForDiff(old []string, new []string) (string, bool) {
	for i, s := range old {
		old[i] = cleanTransForDiff(s)
	}
	for i, s := range new {
		new[i] = cleanTransForDiff(s)
	}
	// var oldIJ = []string{}
	// var newIJ = []string{}
	// for _, s := range old {
	// 	oldIJ = append(oldIJ, cleanTransForIJDiff(s))
	// }
	// for _, s := range new {
	// 	newIJ = append(newIJ, cleanTransForIJDiff(s))
	// }
	if reflect.DeepEqual(old, new) {
		return "ALL EQ", true
	} else if old[0] == new[0] {
		return "#1 EQ", false
		// } else if reflect.DeepEqual(oldIJ, newIJ) {
		// 	return "ALL EQ IJ", false
		// } else if oldIJ[0] == newIJ[0] {
		// 	return "#1 EQ IJ", false
	} else {
		return "DIFF", false
	}
}

func main() {

	var f = flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
	var debug = f.Bool("debug", false, "print extra debug info (default: false)")
	var force = f.Bool("force", false, "print transcriptions even if errors are found (default: false)")
	var column = f.Int("column", 0, "only convert specified column (default: first field)")
	var quiet = f.Bool("quiet", false, "inhibit warnings (default: false)")
	var test = f.Bool("test", false, "test g2p against input file; orth <tab> trans (default: false)")
	removeStress = f.Bool("testremovestress", false, "remove stress when comparing using the -test switch (default: false)")
	var ssFile = f.String("symbolset", "", "use specified symbol set file for validating the symbols in the g2p rule set (default: none; overrides the g2p rule file's symbolset, if any)")
	var help = f.Bool("help", false, "print help message")

	var usage = `go run g2p.go <FLAGS> <G2P RULE FILE> <WORDS (FILES OR LIST OF WORDS)> (optional)

FLAGS:
   -force      bool    print transcriptions even if errors are found (default: false)
   -debug      bool    print extra debug info (default: false)
   -column     string  only convert specified column (default: first field)
   -quiet      bool    inhibit warnings (default: false)
   -test       bool    test g2p against input file; orth <tab> trans (default: false)
   -testremovestress bool remove stress when comparing using the -test switch (default: false)
   -symbolset  string  use specified symbol set file for validating the symbols in the g2p rule set (default: none)
   -help       bool    print help message`

	f.Usage = func() {
		l.Printf(usage)
	}

	var args = os.Args
	if strings.HasSuffix(args[0], "g2p") {
		args = args[1:] // remove first argument if it's the program name
	}
	err := f.Parse(args)
	if err != nil {
		os.Exit(1)
	}

	args = f.Args()

	if *help {
		l.Println(usage)
		os.Exit(1)
	}

	if len(args) < 1 {
		l.Println(usage)
		os.Exit(1)
	}

	rbg2p.Debug = *debug

	g2pFile := args[0]
	ruleSet, err := rbg2p.LoadFile(g2pFile)
	if err != nil {
		l.Printf("couldn't load rule file %s : %s", g2pFile, err)
		os.Exit(1)
	}

	if *ssFile != "" {
		phonemeSet, err := rbg2p.LoadPhonemeSetFile(*ssFile, ruleSet.PhonemeDelimiter)
		if err != nil {
			l.Printf("couldn't load symbol set : %s", err)
			os.Exit(1)
		}
		ruleSet.PhonemeSet = phonemeSet
	}

	haltingError := false
	result := ruleSet.Test()
	for _, e := range result.Errors {
		l.Printf("ERROR: %v\n", e)
	}
	l.Printf("%d ERROR(S) FOR %s\n", len(result.Errors), g2pFile)
	if !*quiet {
		for _, e := range result.Warnings {
			l.Printf("WARNING: %v\n", e)
		}
	}
	l.Printf("%d WARNING(S) FOR %s\n", len(result.Warnings), g2pFile)
	if len(result.Errors) > 0 {
		haltingError = true
	}
	if len(result.FailedTests) > 0 {
		for _, e := range result.FailedTests {
			l.Printf("FAILED TEST: %v\n", e)
		}
		l.Printf("%d OF %d TESTS FAILED FOR %s\n", len(result.FailedTests), len(ruleSet.Tests), g2pFile)
		haltingError = true
	} else {
		l.Printf("ALL %d TESTS PASSED FOR %s\n", len(ruleSet.Tests), g2pFile)
	}

	if haltingError && !*force {
		os.Exit(1)
	}

	nTotal := 0
	nErrs := 0
	nTrans := 0
	nTests := 0
	testRes := make(map[string]int)
	if *test {
		fmt.Println("ORTH\tNEW TRANSES\tOLD TRANSES\tDIFFTAG\t(DIFF)?")
	}
	var processString = func(s string) {
		nTotal = nTotal + 1
		fs := strings.Split(s, "\t")
		o := fs[*column]
		res := transcribe(ruleSet, o)
		if res.result || *force {
			nTrans = nTrans + 1
			if *test {
				refTranses := fs[(*column + 1):]
				nTests++
				info, _ := compareForDiff(res.transes, refTranses)
				testRes[info]++
				outFs := []string{res.orth, strings.Join(res.transes, " # "), strings.Join(refTranses, "#"), info}
				if info == "DIFF" {
					dmp := diffmatchpatch.New()
					diffs := dmp.DiffMain(outFs[1], outFs[2], false)
					diffsOnly := []diffmatchpatch.Diff{}
					diffsOnlyText := []string{}
					for _, d := range diffs {
						if d.Type != diffmatchpatch.DiffEqual {
							diffsOnly = append(diffsOnly, d)
							diffsOnlyText = append(diffsOnlyText, d.Text)
						}
					}
					outFs = append(outFs, dmp.DiffPrettyText(diffs))
					outFs = append(outFs, fmt.Sprintf("%v", diffsOnly))
					outFs = append(outFs, strings.Join(diffsOnlyText, "|"))
				}

				fmt.Println(strings.Join(outFs, "\t"))
			} else {
				print(s, res.orth, res.transes)
			}
		}
		if !res.result {
			nErrs = nErrs + 1
		}
	}

	if len(args) > 1 {
		for i := 1; i < len(args); i++ {
			s := args[i]
			if _, err := os.Stat(s); os.IsNotExist(err) {
				processString(s)
				// nTotal = nTotal + 1
				// res := transcribe(ruleSet, s)
				// if res.result || *force {
				// 	nTrans = nTrans + 1
				// 	fmt.Printf("%s\t%s\n", s, strings.Join(res.transes, "\t"))
				// }
				// if !res.result {
				// 	nErrs = nErrs + 1
				// }
			} else {
				fh, err := os.Open(filepath.Clean(s))
				if err != nil {
					l.Println(err)
					os.Exit(1)
				}
				defer fh.Close()
				sc := bufio.NewScanner(fh)
				for sc.Scan() {
					if err := sc.Err(); err != nil {
						l.Println(err)
						os.Exit(1)
					}
					line := sc.Text()
					if strings.TrimSpace(line) == "" {
						l.Println("Skipping empty line")
						continue
					}
					if strings.HasPrefix(strings.TrimSpace(line), "#") {
						l.Println("Skipping line " + line)
						continue
					}
					processString(line)
					// 	nTotal = nTotal + 1
					// 	fs := strings.Split(line, "\t")
					// 	o, refTranses := fs[0], fs[1:]
					// 	res := transcribe(ruleSet, o)
					// 	if res.result || *force {
					// 		nTrans = nTrans + 1
					// 		if *test {
					// 			nTests++
					// 			info, _ := compareForDiff(res.transes, refTranses)
					// 			testRes[info]++
					// 			outFs := []string{res.orth, strings.Join(res.transes, " # "), strings.Join(refTranses, "#"), info}
					// 			if info == "DIFF" {
					// 				dmp := diffmatchpatch.New()
					// 				diffs := dmp.DiffMain(outFs[1], outFs[2], false)
					// 				diffsOnly := []diffmatchpatch.Diff{}
					// 				diffsOnlyText := []string{}
					// 				for _, d := range diffs {
					// 					if d.Type != diffmatchpatch.DiffEqual {
					// 						diffsOnly = append(diffsOnly, d)
					// 						diffsOnlyText = append(diffsOnlyText, d.Text)
					// 					}
					// 				}
					// 				outFs = append(outFs, dmp.DiffPrettyText(diffs))
					// 				outFs = append(outFs, fmt.Sprintf("%v", diffsOnly))
					// 				outFs = append(outFs, strings.Join(diffsOnlyText, "|"))
					// 			}

					// 			fmt.Println(strings.Join(outFs, "\t"))
					// 		} else {
					// 			print(res.orth, res.transes)
					// 		}
					// 	}
					// 	if !res.result {
					// 		nErrs = nErrs + 1
					// 	}
				}
			}
		}
	} else {
		fmt.Fprintf(os.Stderr, "Reading input from stdin...\n")
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			line := scanner.Text()
			if strings.TrimSpace(line) == "" {
				l.Println("Skipping empty line")
				continue
			}
			if strings.HasPrefix(strings.TrimSpace(line), "#") {
				l.Println("Skipping line " + line)
				continue
			}
			processString(line)
		}
	}
	l.Printf("%-18s: % 7d", "TOTAL INPUT", nTotal)
	l.Printf("%-18s: % 7d", "ERRORS", nErrs)
	l.Printf("%-18s: % 7d", "TRANSCRIBED", nTrans)
	if *test {
		l.Printf("%-18s: % 7d", "TESTED", nTests)
		var keys []string
		for k := range testRes {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for _, tag := range keys {
			freq := testRes[tag]
			s := " > TEST " + tag
			l.Printf("%-18s: % 7d", s, freq)
		}
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/open-falcon/falcon-plus/g"
	"github.com/spf13/cobra"
)

var Monitor = &cobra.Command{
	Use:   "monitor [Module ...]",
	Short: "Display an Open-Falcon module's log",
	Long: `
Display the log of the specified Open-Falcon module.
A module represents a single node in a cluster.
Modules:
  ` + strings.Join(g.AllModulesInOrder, " "),
	RunE: monitor,
}

func checkMonReq(name string) error {
	if !g.HasModule(name) {
		return fmt.Errorf("%s doesn't exist", name)
	}

	if !g.HasLogfile(name) {
		r := g.Rel(g.Cfg(name))
		return fmt.Errorf("expect logfile: %s", r)
	}

	return nil
}

func monitor(c *cobra.Command, args []string) error {
	if len(args) < 1 {
		return c.Usage()
	}
	var tailArgs []string = []string{"-f"}
	for _,moduleName := range args {
		if err := checkMonReq(moduleName); err != nil {
			return err
		}

		tailArgs = append(tailArgs, g.LogPath(moduleName))
	}
	cmd := exec.Command("tail", tailArgs...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}
<commit_msg>code fmt check error fix<commit_after>package cmd

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/open-falcon/falcon-plus/g"
	"github.com/spf13/cobra"
)

var Monitor = &cobra.Command{
	Use:   "monitor [Module ...]",
	Short: "Display an Open-Falcon module's log",
	Long: `
Display the log of the specified Open-Falcon module.
A module represents a single node in a cluster.
Modules:
  ` + strings.Join(g.AllModulesInOrder, " "),
	RunE: monitor,
}

func checkMonReq(name string) error {
	if !g.HasModule(name) {
		return fmt.Errorf("%s doesn't exist", name)
	}

	if !g.HasLogfile(name) {
		r := g.Rel(g.Cfg(name))
		return fmt.Errorf("expect logfile: %s", r)
	}

	return nil
}

func monitor(c *cobra.Command, args []string) error {
	if len(args) < 1 {
		return c.Usage()
	}
	var tailArgs []string = []string{"-f"}
	for _, moduleName := range args {
		if err := checkMonReq(moduleName); err != nil {
			return err
		}

		tailArgs = append(tailArgs, g.LogPath(moduleName))
	}
	cmd := exec.Command("tail", tailArgs...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}
<|endoftext|>


<commit_before>package metric

import (
	"fmt"
	"github.com/Knetic/govaluate"
	"github.com/Sirupsen/logrus"
	"github.com/soniah/gosnmp"
	"github.com/toni-moreno/snmpcollector/pkg/config"
	"github.com/toni-moreno/snmpcollector/pkg/data/measurement/filter"
	"github.com/toni-moreno/snmpcollector/pkg/data/snmp"
	"math"
	"regexp"
	"strconv"
	"time"
)

var (
	confDir string              //Needed to get File Filters data
	dbc     *config.DatabaseCfg //Needed to get Custom Filter  data
)

// SetConfDir  enable load File Filters from anywhere in the our FS.
func SetConfDir(dir string) {
	confDir = dir
}

// SetDB load database config to load data if needed (used in filters)
func SetDB(db *config.DatabaseCfg) {
	dbc = db
}

const (
	NeverReport     = 0
	AlwaysReport    = 1
	OnNonZeroReport = 2
	OnChangedReport = 3
)

//SnmpMetric type to metric runtime
type SnmpMetric struct {
	cfg         *config.SnmpMetricCfg
	ID          string
	CookedValue interface{}
	CurValue    interface{}
	LastValue   interface{}
	CurTime     time.Time
	LastTime    time.Time
	ElapsedTime float64
	Compute     func(arg ...interface{})                `json:"-"`
	Scale       func()                                  `json:"-"`
	SetRawData  func(pdu gosnmp.SnmpPDU, now time.Time) `json:"-"`
	RealOID     string
	Report      int //if false this metric won't be sent to the ouput buffer (is just taken as a coomputed input for other metrics)
	//for STRINGPARSER
	re   *regexp.Regexp
	expr *govaluate.EvaluableExpression
	//for CONDITIONEVAL
	condflt filter.Filter
	// Logger
	log *logrus.Logger
}

// GetDataSrcType get needed data
func (s *SnmpMetric) GetDataSrcType() string {
	return s.cfg.DataSrcType
}

// PrintDebugCfg helps users get data about metric configuration
func (s *SnmpMetric) PrintDebugCfg() {
	s.log.Debugf("DEBUG METRIC  CONFIG %+v", s.cfg)
}

// IsTag needed to generate Influx measurements
func (s *SnmpMetric) IsTag() bool {
	return s.cfg.IsTag
}

// GetFieldName  needed to generate Influx measurements
func (s *SnmpMetric) GetFieldName() string {
	return s.cfg.FieldName
}

// New constructor
func New(c *config.SnmpMetricCfg) (*SnmpMetric, error) {
	metric := &SnmpMetric{}
	err := metric.Init(c)
	return metric, err
}

func NewWithLog(c *config.SnmpMetricCfg, l *logrus.Logger) (*SnmpMetric, error) {
	metric := &SnmpMetric{log: l}
	err := metric.Init(c)
	return metric, err
}

func (s *SnmpMetric) SetLogger(l *logrus.Logger) {
	s.log = l
}

func (s *SnmpMetric) Init(c *config.SnmpMetricCfg) error {
	if c == nil {
		return fmt.Errorf("Error on initialice device, configuration struct is nil")
	}
	s.cfg = c
	s.RealOID = c.BaseOID
	s.ID = s.cfg.ID
	if s.cfg.Scale != 0.0 || s.cfg.Shift != 0.0 {
		s.Scale = func() {
			s.CookedValue = (s.cfg.Scale * float64(s.CookedValue.(float64))) + s.cfg.Shift
		}
	} else {
		s.Scale = func() {
		}
	}
	switch s.cfg.DataSrcType {
	case "CONDITIONEVAL":
		//select
		cond, err := dbc.GetOidConditionCfgByID(s.cfg.ExtraData)
		if err != nil {
			s.log.Errorf("Error getting CONDITIONEVAL [id: %s ] data : %s", s.cfg.ExtraData, err)
		}
		//get Regexp
		s.condflt = filter.NewOidFilter(cond.OIDCond, cond.CondType, cond.CondValue, s.log)

		s.Compute = func(arg ...interface{}) {
			//walk := arg[0].(func(string, gosnmp.WalkFunc) error)
			//err := s.condflt.Init(walk)
			s.condflt.Init(arg...)
			s.condflt.Update()
			s.CookedValue = s.condflt.Count()
			s.CurTime = time.Now()
			s.Scale()
		}
		//Sign
		//set Process Data
	case "TIMETICKS": //Cooked TimeTicks
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			val := snmp.PduVal2Int64(pdu)
			s.CookedValue = float64(val / 100) //now data in secoonds
			s.CurTime = now
			s.Scale()
		}

		//Signed Integers
	case "INTEGER", "Integer32":
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			val := snmp.PduVal2Int64(pdu)
			s.CookedValue = float64(val)
			s.CurTime = now
			s.Scale()
		}
		//Unsigned Integers
	case "Counter32", "Gauge32", "Counter64", "TimeTicks", "UInteger32", "Unsigned32":
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			val := snmp.PduVal2UInt64(pdu)
			s.CookedValue = float64(val)
			s.CurTime = now
			s.Scale()
		}
	case "COUNTER32": //Increment computed
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			//first time only set values and reassign itself to the complete method this will avoi to send invalid data
			val := snmp.PduVal2UInt64(pdu)
			s.CurValue = val
			s.CurTime = now
			s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
				val := snmp.PduVal2UInt64(pdu)
				s.LastTime = s.CurTime
				s.LastValue = s.CurValue
				s.CurValue = val
				s.CurTime = now
				s.Compute()
				s.Scale()
			}
		}
		if s.cfg.GetRate == true {
			s.Compute = func(arg ...interface{}) {
				s.ElapsedTime = s.CurTime.Sub(s.LastTime).Seconds()
				if s.CurValue.(uint64) < s.LastValue.(uint64) {
					s.CookedValue = float64(math.MaxInt32-s.LastValue.(uint64)+s.CurValue.(uint64)) / s.ElapsedTime
				} else {
					s.CookedValue = float64(s.CurValue.(uint64)-s.LastValue.(uint64)) / s.ElapsedTime
				}
			}
		} else {
			s.Compute = func(arg ...interface{}) {
				s.ElapsedTime = s.CurTime.Sub(s.LastTime).Seconds()
				if s.CurValue.(uint64) < s.LastValue.(uint64) {
					s.CookedValue = float64(math.MaxInt32 - s.LastValue.(uint64) + s.CurValue.(uint64))
				} else {
					s.CookedValue = float64(s.CurValue.(uint64) - s.LastValue.(uint64))
				}
			}
		}
	case "COUNTER64": //Increment computed
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			//log.Debugf("========================================>COUNTER64: first time :%s ", s.RealOID)
			//first time only set values and reassign itself to the complete method
			val := snmp.PduVal2UInt64(pdu)
			s.CurValue = val
			s.CurTime = now
			s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
				//log.Debugf("========================================>COUNTER64: the other time:%s", s.RealOID)
				val := snmp.PduVal2UInt64(pdu)
				s.LastTime = s.CurTime
				s.LastValue = s.CurValue
				s.CurValue = val
				s.CurTime = now
				s.Compute()
				s.Scale()
			}
		}
		if s.cfg.GetRate == true {
			s.Compute = func(arg ...interface{}) {
				s.ElapsedTime = s.CurTime.Sub(s.LastTime).Seconds()
				//duration := s.CurTime.Sub(s.LastTime)
				if s.CurValue.(uint64) < s.LastValue.(uint64) {
					s.CookedValue = float64(math.MaxInt64-s.LastValue.(uint64)+s.CurValue.(uint64)) / s.ElapsedTime
				} else {
					s.CookedValue = float64(s.CurValue.(uint64)-s.LastValue.(uint64)) / s.ElapsedTime
				}
			}
		} else {
			s.Compute = func(arg ...interface{}) {
				s.ElapsedTime = s.CurTime.Sub(s.LastTime).Seconds()
				if s.CurValue.(uint64) < s.LastValue.(uint64) {
					s.CookedValue = float64(math.MaxInt64 - s.LastValue.(uint64) + s.CurValue.(uint64))
				} else {
					s.CookedValue = float64(s.CurValue.(uint64) - s.LastValue.(uint64))
				}
			}

		}
	case "OCTETSTRING":
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			s.CookedValue = snmp.PduVal2str(pdu)
			s.CurTime = now
		}
	case "IpAddress":
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			s.CookedValue, _ = snmp.PduVal2IPaddr(pdu)
			s.CurTime = now
		}
	case "HWADDR":
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			s.CookedValue, _ = snmp.PduVal2Hwaddr(pdu)
			s.CurTime = now
		}
	case "STRINGPARSER":
		//get Regexp
		re, err := regexp.Compile(s.cfg.ExtraData)
		if err != nil {
			return fmt.Errorf("Error on initialice STRINGPARSER, invalind Regular Expression : %s", s.cfg.ExtraData)
		}
		s.re = re
		//set Process Data
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			str := snmp.PduVal2str(pdu)
			retarray := s.re.FindStringSubmatch(str)
			if len(retarray) < 2 {
				s.log.Warnf("Error for metric [%s] parsing REGEXG [%s] on string [%s] without capturing group", s.cfg.ID, s.cfg.ExtraData, str)
				return
			}
			//retarray[0] contains full string
			if len(retarray[1]) == 0 {
				s.log.Warnf("Error for metric [%s] parsing REGEXG [%s] on string [%s] cause  void capturing group", s.cfg.ID, s.cfg.ExtraData, str)
				return
			}
			value, err := strconv.ParseFloat(retarray[1], 64)
			if err != nil {
				s.log.Warnf("Error parsing float for metric %s : error: %s", s.cfg.ID, err)
				return
			}
			s.CookedValue = value
			s.CurTime = now
			s.Scale()
		}
	case "STRINGEVAL":

		expression, err := govaluate.NewEvaluableExpression(s.cfg.ExtraData)
		if err != nil {
			s.log.Errorf("Error on initialice STRINGEVAL, evaluation : %s : ERROR : %s", s.cfg.ExtraData, err)
			return err
		}
		s.expr = expression
		//set Process Data
		s.Compute = func(arg ...interface{}) {
			//parameters := make(map[string]interface{})
			parameters := arg[0].(map[string]interface{})
			result, err := s.expr.Evaluate(parameters)
			if err != nil {
				s.log.Errorf("Error in metric %s On EVAL string: %s : ERROR : %s", s.cfg.ID, s.cfg.ExtraData, err)
				return
			}
			//Influxdb has not support for NaN,Inf values
			//https://github.com/influxdata/influxdb/issues/4089
			switch v := result.(type) {
			case float64:
				if math.IsNaN(v) || math.IsInf(v, 0) {
					s.log.Warnf("Warning in metric %s On EVAL string: %s : Value is not a valid Floating Pint (NaN/Inf) : %f", s.cfg.ID, s.cfg.ExtraData, v)
					return
				}
			}
			s.CookedValue = result
			s.CurTime = time.Now()
			s.Scale()
		}
	}
	return nil
}
<commit_msg>fix bad counter limits<commit_after>package metric

import (
	"fmt"
	"github.com/Knetic/govaluate"
	"github.com/Sirupsen/logrus"
	"github.com/soniah/gosnmp"
	"github.com/toni-moreno/snmpcollector/pkg/config"
	"github.com/toni-moreno/snmpcollector/pkg/data/measurement/filter"
	"github.com/toni-moreno/snmpcollector/pkg/data/snmp"
	"math"
	"regexp"
	"strconv"
	"time"
)

var (
	confDir string              //Needed to get File Filters data
	dbc     *config.DatabaseCfg //Needed to get Custom Filter  data
)

// SetConfDir  enable load File Filters from anywhere in the our FS.
func SetConfDir(dir string) {
	confDir = dir
}

// SetDB load database config to load data if needed (used in filters)
func SetDB(db *config.DatabaseCfg) {
	dbc = db
}

const (
	NeverReport     = 0
	AlwaysReport    = 1
	OnNonZeroReport = 2
	OnChangedReport = 3
)

//SnmpMetric type to metric runtime
type SnmpMetric struct {
	cfg         *config.SnmpMetricCfg
	ID          string
	CookedValue interface{}
	CurValue    interface{}
	LastValue   interface{}
	CurTime     time.Time
	LastTime    time.Time
	ElapsedTime float64
	Compute     func(arg ...interface{})                `json:"-"`
	Scale       func()                                  `json:"-"`
	SetRawData  func(pdu gosnmp.SnmpPDU, now time.Time) `json:"-"`
	RealOID     string
	Report      int //if false this metric won't be sent to the ouput buffer (is just taken as a coomputed input for other metrics)
	//for STRINGPARSER
	re   *regexp.Regexp
	expr *govaluate.EvaluableExpression
	//for CONDITIONEVAL
	condflt filter.Filter
	// Logger
	log *logrus.Logger
}

// GetDataSrcType get needed data
func (s *SnmpMetric) GetDataSrcType() string {
	return s.cfg.DataSrcType
}

// PrintDebugCfg helps users get data about metric configuration
func (s *SnmpMetric) PrintDebugCfg() {
	s.log.Debugf("DEBUG METRIC  CONFIG %+v", s.cfg)
}

// IsTag needed to generate Influx measurements
func (s *SnmpMetric) IsTag() bool {
	return s.cfg.IsTag
}

// GetFieldName  needed to generate Influx measurements
func (s *SnmpMetric) GetFieldName() string {
	return s.cfg.FieldName
}

// New constructor
func New(c *config.SnmpMetricCfg) (*SnmpMetric, error) {
	metric := &SnmpMetric{}
	err := metric.Init(c)
	return metric, err
}

func NewWithLog(c *config.SnmpMetricCfg, l *logrus.Logger) (*SnmpMetric, error) {
	metric := &SnmpMetric{log: l}
	err := metric.Init(c)
	return metric, err
}

func (s *SnmpMetric) SetLogger(l *logrus.Logger) {
	s.log = l
}

func (s *SnmpMetric) Init(c *config.SnmpMetricCfg) error {
	if c == nil {
		return fmt.Errorf("Error on initialice device, configuration struct is nil")
	}
	s.cfg = c
	s.RealOID = c.BaseOID
	s.ID = s.cfg.ID
	if s.cfg.Scale != 0.0 || s.cfg.Shift != 0.0 {
		s.Scale = func() {
			s.CookedValue = (s.cfg.Scale * float64(s.CookedValue.(float64))) + s.cfg.Shift
		}
	} else {
		s.Scale = func() {
		}
	}
	switch s.cfg.DataSrcType {
	case "CONDITIONEVAL":
		//select
		cond, err := dbc.GetOidConditionCfgByID(s.cfg.ExtraData)
		if err != nil {
			s.log.Errorf("Error getting CONDITIONEVAL [id: %s ] data : %s", s.cfg.ExtraData, err)
		}
		//get Regexp
		s.condflt = filter.NewOidFilter(cond.OIDCond, cond.CondType, cond.CondValue, s.log)

		s.Compute = func(arg ...interface{}) {
			s.condflt.Init(arg...)
			s.condflt.Update()
			s.CookedValue = s.condflt.Count()
			s.CurTime = time.Now()
			s.Scale()
		}
		//Sign
		//set Process Data
	case "TIMETICKS": //Cooked TimeTicks
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			val := snmp.PduVal2Int64(pdu)
			s.CookedValue = float64(val / 100) //now data in secoonds
			s.CurTime = now
			s.Scale()
		}

		//Signed Integers
	case "INTEGER", "Integer32":
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			val := snmp.PduVal2Int64(pdu)
			s.CookedValue = float64(val)
			s.CurTime = now
			s.Scale()
		}
		//Unsigned Integers
	case "Counter32", "Gauge32", "Counter64", "TimeTicks", "UInteger32", "Unsigned32":
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			val := snmp.PduVal2UInt64(pdu)
			s.CookedValue = float64(val)
			s.CurTime = now
			s.Scale()
		}
	case "COUNTER32": //Increment computed
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			//first time only set values and reassign itself to the complete method this will avoi to send invalid data
			val := snmp.PduVal2UInt64(pdu)
			s.CurValue = val
			s.CurTime = now
			s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
				val := snmp.PduVal2UInt64(pdu)
				s.LastTime = s.CurTime
				s.LastValue = s.CurValue
				s.CurValue = val
				s.CurTime = now
				s.Compute()
				s.Scale()
			}
		}
		if s.cfg.GetRate == true {
			s.Compute = func(arg ...interface{}) {
				s.ElapsedTime = s.CurTime.Sub(s.LastTime).Seconds()
				if s.CurValue.(uint64) < s.LastValue.(uint64) {
					s.CookedValue = float64(math.MaxUint32-s.LastValue.(uint64)+s.CurValue.(uint64)) / s.ElapsedTime
				} else {
					s.CookedValue = float64(s.CurValue.(uint64)-s.LastValue.(uint64)) / s.ElapsedTime
				}
			}
		} else {
			s.Compute = func(arg ...interface{}) {
				s.ElapsedTime = s.CurTime.Sub(s.LastTime).Seconds()
				if s.CurValue.(uint64) < s.LastValue.(uint64) {
					s.CookedValue = float64(math.MaxUint32 - s.LastValue.(uint64) + s.CurValue.(uint64))
				} else {
					s.CookedValue = float64(s.CurValue.(uint64) - s.LastValue.(uint64))
				}
			}
		}
	case "COUNTER64": //Increment computed
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			//log.Debugf("========================================>COUNTER64: first time :%s ", s.RealOID)
			//first time only set values and reassign itself to the complete method
			val := snmp.PduVal2UInt64(pdu)
			s.CurValue = val
			s.CurTime = now
			s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
				//log.Debugf("========================================>COUNTER64: the other time:%s", s.RealOID)
				val := snmp.PduVal2UInt64(pdu)
				s.LastTime = s.CurTime
				s.LastValue = s.CurValue
				s.CurValue = val
				s.CurTime = now
				s.Compute()
				s.Scale()
			}
		}
		if s.cfg.GetRate == true {
			s.Compute = func(arg ...interface{}) {
				s.ElapsedTime = s.CurTime.Sub(s.LastTime).Seconds()
				//duration := s.CurTime.Sub(s.LastTime)
				if s.CurValue.(uint64) < s.LastValue.(uint64) {
					s.CookedValue = float64(math.MaxUint64-s.LastValue.(uint64)+s.CurValue.(uint64)) / s.ElapsedTime
				} else {
					s.CookedValue = float64(s.CurValue.(uint64)-s.LastValue.(uint64)) / s.ElapsedTime
				}
			}
		} else {
			s.Compute = func(arg ...interface{}) {
				s.ElapsedTime = s.CurTime.Sub(s.LastTime).Seconds()
				if s.CurValue.(uint64) < s.LastValue.(uint64) {
					s.CookedValue = float64(math.MaxUint64 - s.LastValue.(uint64) + s.CurValue.(uint64))
				} else {
					s.CookedValue = float64(s.CurValue.(uint64) - s.LastValue.(uint64))
				}
			}

		}
	case "OCTETSTRING":
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			s.CookedValue = snmp.PduVal2str(pdu)
			s.CurTime = now
		}
	case "IpAddress":
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			s.CookedValue, _ = snmp.PduVal2IPaddr(pdu)
			s.CurTime = now
		}
	case "HWADDR":
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			s.CookedValue, _ = snmp.PduVal2Hwaddr(pdu)
			s.CurTime = now
		}
	case "STRINGPARSER":
		//get Regexp
		re, err := regexp.Compile(s.cfg.ExtraData)
		if err != nil {
			return fmt.Errorf("Error on initialice STRINGPARSER, invalind Regular Expression : %s", s.cfg.ExtraData)
		}
		s.re = re
		//set Process Data
		s.SetRawData = func(pdu gosnmp.SnmpPDU, now time.Time) {
			str := snmp.PduVal2str(pdu)
			retarray := s.re.FindStringSubmatch(str)
			if len(retarray) < 2 {
				s.log.Warnf("Error for metric [%s] parsing REGEXG [%s] on string [%s] without capturing group", s.cfg.ID, s.cfg.ExtraData, str)
				return
			}
			//retarray[0] contains full string
			if len(retarray[1]) == 0 {
				s.log.Warnf("Error for metric [%s] parsing REGEXG [%s] on string [%s] cause  void capturing group", s.cfg.ID, s.cfg.ExtraData, str)
				return
			}
			value, err := strconv.ParseFloat(retarray[1], 64)
			if err != nil {
				s.log.Warnf("Error parsing float for metric %s : error: %s", s.cfg.ID, err)
				return
			}
			s.CookedValue = value
			s.CurTime = now
			s.Scale()
		}
	case "STRINGEVAL":

		expression, err := govaluate.NewEvaluableExpression(s.cfg.ExtraData)
		if err != nil {
			s.log.Errorf("Error on initialice STRINGEVAL, evaluation : %s : ERROR : %s", s.cfg.ExtraData, err)
			return err
		}
		s.expr = expression
		//set Process Data
		s.Compute = func(arg ...interface{}) {
			//parameters := make(map[string]interface{})
			parameters := arg[0].(map[string]interface{})
			result, err := s.expr.Evaluate(parameters)
			if err != nil {
				s.log.Errorf("Error in metric %s On EVAL string: %s : ERROR : %s", s.cfg.ID, s.cfg.ExtraData, err)
				return
			}
			//Influxdb has not support for NaN,Inf values
			//https://github.com/influxdata/influxdb/issues/4089
			switch v := result.(type) {
			case float64:
				if math.IsNaN(v) || math.IsInf(v, 0) {
					s.log.Warnf("Warning in metric %s On EVAL string: %s : Value is not a valid Floating Pint (NaN/Inf) : %f", s.cfg.ID, s.cfg.ExtraData, v)
					return
				}
			}
			s.CookedValue = result
			s.CurTime = time.Now()
			s.Scale()
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package compute

import (
	"encoding/json"
	"fmt"
	"net/http"
)

// Tag represents a tag applied to an asset.
type Tag struct {
	Name  string `json:"tagKeyName"`
	Value string `json:"tagKeyValue"`
}

// TagDetail represents detailed information about a tag applied to an asset.
type TagDetail struct {
	AssetType        string `json:"assetType"`
	AssetID          string `json:"assetId"`
	AssetName        string `json:"assetName"`
	DataCenterID     string `json:"datacenterId"`
	TagKeyID         string `json:"tagKeyId"`
	Name             string `json:"tagKeyName"`
	Value            string `json:"value"`
	IsValueRequired  bool   `json:"valueRequired"`
	DisplayOnReports bool   `json:"displayOnReport"`
}

// TagDetails represents a page of TagDetail results.
type TagDetails struct {
	Items []TagDetail

	PagedResult
}

// Request body when applying tags to an asset.
type applyTags struct {
	AssetType string `json:"assetType"`
	AssetID   string `json:"assetId"`
	Tags      []Tag  `json:"tag"`
}

// Request body when removing tags from an asset.
type removeTags struct {
	AssetType string   `json:"assetType"`
	AssetID   string   `json:"assetId"`
	TagNames  []string `json:"tagKeyName"`
}

// TagKey represents a key for asset tags.
type TagKey struct {
	ID string `json:"id"`

	tagKey
}

// TagKeys represents a page of TagKey results.
type TagKeys struct {
	Items []TagKey

	PagedResult
}

// Common fields for a tag key.
type tagKey struct {
	Name             string `json:"name"`
	Description      string `json:"description"`
	IsValueRequired  bool   `json:"valueRequired"`
	DisplayOnReports bool   `json:"displayOnReport"`
}

// Request body for deleting a tag key.
type deleteTagKey struct {
	ID string `json:"id"`
}

// GetAssetTags gets all tags applied to the specified asset.
func (client *Client) GetAssetTags(assetID string, assetType string, paging *PagingInfo) (tags *TagDetails, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/tag/tag?assetId=%s&assetType=%s", organizationID, assetID, assetType)
	if paging != nil {
		requestURI += fmt.Sprintf("&pageNumber=%d&pageSize=%d", paging.PageNumber, paging.PageSize)
	}
	request, err := client.newRequestV22(requestURI, http.MethodGet, nil)
	if err != nil {
		return nil, err
	}

	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	if statusCode != http.StatusOK {
		var apiResponse *APIResponseV2

		apiResponse, err = readAPIResponseAsJSON(responseBody, statusCode)
		if err != nil {
			return nil, err
		}

		return nil, apiResponse.ToError("Request failed with status code %d (%s): %s", statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	tags = &TagDetails{}
	err = json.Unmarshal(responseBody, tags)

	return tags, err
}

// ApplyAssetTags applies the specified tags to an asset.
func (client *Client) ApplyAssetTags(assetID string, assetType string, tags ...Tag) (response *APIResponseV2, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/tag/applyTags", organizationID)
	request, err := client.newRequestV22(requestURI, http.MethodPost, &applyTags{
		AssetID:   assetID,
		AssetType: assetType,
		Tags:      tags,
	})
	if err != nil {
		return nil, err
	}
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	return readAPIResponseAsJSON(responseBody, statusCode)
}

// RemoveAssetTags removes the specified tags from an asset.
func (client *Client) RemoveAssetTags(assetID string, assetType string, tagNames ...string) (response *APIResponseV2, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/tag/removeTags", organizationID)
	request, err := client.newRequestV22(requestURI, http.MethodPost, &removeTags{
		AssetID:   assetID,
		AssetType: assetType,
		TagNames:  tagNames,
	})
	if err != nil {
		return nil, err
	}
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	return readAPIResponseAsJSON(responseBody, statusCode)
}

// GetTagKey retrieves the tag key with the specified Id.
// Returns nil if no tag key is found with the specified Id.
func (client *Client) GetTagKey(id string) (tagKey *TagKey, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/tags/tagKey/%s", organizationID, id)
	request, err := client.newRequestV22(requestURI, http.MethodGet, nil)
	if err != nil {
		return nil, err
	}
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	if statusCode != http.StatusOK {
		var apiResponse *APIResponseV2

		apiResponse, err = readAPIResponseAsJSON(responseBody, statusCode)
		if err != nil {
			return nil, err
		}

		if apiResponse.ResponseCode == ResponseCodeResourceNotFound {
			return nil, nil // Not an error, but was not found.
		}

		return nil, apiResponse.ToError("Request to retrieve tag key '%s' failed with status code %d (%s): %s", id, statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	tagKey = &TagKey{}
	err = json.Unmarshal(responseBody, tagKey)
	if err != nil {
		return nil, err
	}

	return tagKey, nil
}

// ListTagKeys lists all tag keys that apply to the specified network domain.
func (client *Client) ListTagKeys(pageNumber int, pageSize int) (tagKeys *TagKeys, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/tag/tagKey?orderBy=name&pageNumber=%d&pageSize=%d", organizationID, pageNumber, pageSize)
	request, err := client.newRequestV22(requestURI, http.MethodGet, nil)
	if err != nil {
		return nil, err
	}

	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	if statusCode != http.StatusOK {
		var apiResponse *APIResponseV2

		apiResponse, err = readAPIResponseAsJSON(responseBody, statusCode)
		if err != nil {
			return nil, err
		}

		return nil, apiResponse.ToError("Request to list tag keys failed with status code %d (%s): %s", statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	tagKeys = &TagKeys{}
	err = json.Unmarshal(responseBody, tagKeys)

	return tagKeys, err
}

// CreateTagKey creates a new tag key.
func (client *Client) CreateTagKey(name string, description string, isValueRequired bool, displayOnReports bool) (tagKeyID string, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return "", err
	}

	requestURI := fmt.Sprintf("%s/tag/createTagKey", organizationID)
	request, err := client.newRequestV22(requestURI, http.MethodPost, &tagKey{
		Name:             name,
		Description:      description,
		IsValueRequired:  isValueRequired,
		DisplayOnReports: displayOnReports,
	})
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return "", err
	}

	apiResponse, err := readAPIResponseAsJSON(responseBody, statusCode)
	if err != nil {
		return "", err
	}

	if apiResponse.ResponseCode != ResponseCodeOK {
		return "", apiResponse.ToError("Request to create tag key '%s' failed with unexpected status code %d (%s): %s", name, statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	// Expected: "info" { "name": "tagKeyId", "value": "the-Id-of-the-new-tag-key" }
	if len(apiResponse.FieldMessages) != 1 || apiResponse.FieldMessages[0].FieldName != "tagKeyId" {
		return "", apiResponse.ToError("Received an unexpected response (missing 'tagKeyId') with status code %d (%s): %s", statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	return apiResponse.FieldMessages[0].Message, nil
}

// DeleteTagKey deletes the specified TagKey rule.
func (client *Client) DeleteTagKey(id string) error {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return err
	}

	requestURI := fmt.Sprintf("%s/tag/deleteTagKey", organizationID)
	request, err := client.newRequestV22(requestURI, http.MethodPost,
		&deleteTagKey{id},
	)
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return err
	}

	apiResponse, err := readAPIResponseAsJSON(responseBody, statusCode)
	if err != nil {
		return err
	}

	if apiResponse.ResponseCode != ResponseCodeOK {
		return apiResponse.ToError("Request to delete tag key '%s' failed with unexpected status code %d (%s): %s", id, statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	return nil
}
<commit_msg>Fix incorrect field name for tag value.<commit_after>package compute

import (
	"encoding/json"
	"fmt"
	"net/http"
)

// Tag represents a tag applied to an asset.
type Tag struct {
	Name  string `json:"tagKeyName"`
	Value string `json:"value"`
}

// TagDetail represents detailed information about a tag applied to an asset.
type TagDetail struct {
	AssetType        string `json:"assetType"`
	AssetID          string `json:"assetId"`
	AssetName        string `json:"assetName"`
	DataCenterID     string `json:"datacenterId"`
	TagKeyID         string `json:"tagKeyId"`
	Name             string `json:"tagKeyName"`
	Value            string `json:"value"`
	IsValueRequired  bool   `json:"valueRequired"`
	DisplayOnReports bool   `json:"displayOnReport"`
}

// TagDetails represents a page of TagDetail results.
type TagDetails struct {
	Items []TagDetail

	PagedResult
}

// Request body when applying tags to an asset.
type applyTags struct {
	AssetType string `json:"assetType"`
	AssetID   string `json:"assetId"`
	Tags      []Tag  `json:"tag"`
}

// Request body when removing tags from an asset.
type removeTags struct {
	AssetType string   `json:"assetType"`
	AssetID   string   `json:"assetId"`
	TagNames  []string `json:"tagKeyName"`
}

// TagKey represents a key for asset tags.
type TagKey struct {
	ID string `json:"id"`

	tagKey
}

// TagKeys represents a page of TagKey results.
type TagKeys struct {
	Items []TagKey

	PagedResult
}

// Common fields for a tag key.
type tagKey struct {
	Name             string `json:"name"`
	Description      string `json:"description"`
	IsValueRequired  bool   `json:"valueRequired"`
	DisplayOnReports bool   `json:"displayOnReport"`
}

// Request body for deleting a tag key.
type deleteTagKey struct {
	ID string `json:"id"`
}

// GetAssetTags gets all tags applied to the specified asset.
func (client *Client) GetAssetTags(assetID string, assetType string, paging *PagingInfo) (tags *TagDetails, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/tag/tag?assetId=%s&assetType=%s", organizationID, assetID, assetType)
	if paging != nil {
		requestURI += fmt.Sprintf("&pageNumber=%d&pageSize=%d", paging.PageNumber, paging.PageSize)
	}
	request, err := client.newRequestV22(requestURI, http.MethodGet, nil)
	if err != nil {
		return nil, err
	}

	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	if statusCode != http.StatusOK {
		var apiResponse *APIResponseV2

		apiResponse, err = readAPIResponseAsJSON(responseBody, statusCode)
		if err != nil {
			return nil, err
		}

		return nil, apiResponse.ToError("Request failed with status code %d (%s): %s", statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	tags = &TagDetails{}
	err = json.Unmarshal(responseBody, tags)

	return tags, err
}

// ApplyAssetTags applies the specified tags to an asset.
func (client *Client) ApplyAssetTags(assetID string, assetType string, tags ...Tag) (response *APIResponseV2, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/tag/applyTags", organizationID)
	request, err := client.newRequestV22(requestURI, http.MethodPost, &applyTags{
		AssetID:   assetID,
		AssetType: assetType,
		Tags:      tags,
	})
	if err != nil {
		return nil, err
	}
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	return readAPIResponseAsJSON(responseBody, statusCode)
}

// RemoveAssetTags removes the specified tags from an asset.
func (client *Client) RemoveAssetTags(assetID string, assetType string, tagNames ...string) (response *APIResponseV2, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/tag/removeTags", organizationID)
	request, err := client.newRequestV22(requestURI, http.MethodPost, &removeTags{
		AssetID:   assetID,
		AssetType: assetType,
		TagNames:  tagNames,
	})
	if err != nil {
		return nil, err
	}
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	return readAPIResponseAsJSON(responseBody, statusCode)
}

// GetTagKey retrieves the tag key with the specified Id.
// Returns nil if no tag key is found with the specified Id.
func (client *Client) GetTagKey(id string) (tagKey *TagKey, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/tags/tagKey/%s", organizationID, id)
	request, err := client.newRequestV22(requestURI, http.MethodGet, nil)
	if err != nil {
		return nil, err
	}
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	if statusCode != http.StatusOK {
		var apiResponse *APIResponseV2

		apiResponse, err = readAPIResponseAsJSON(responseBody, statusCode)
		if err != nil {
			return nil, err
		}

		if apiResponse.ResponseCode == ResponseCodeResourceNotFound {
			return nil, nil // Not an error, but was not found.
		}

		return nil, apiResponse.ToError("Request to retrieve tag key '%s' failed with status code %d (%s): %s", id, statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	tagKey = &TagKey{}
	err = json.Unmarshal(responseBody, tagKey)
	if err != nil {
		return nil, err
	}

	return tagKey, nil
}

// ListTagKeys lists all tag keys that apply to the specified network domain.
func (client *Client) ListTagKeys(pageNumber int, pageSize int) (tagKeys *TagKeys, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/tag/tagKey?orderBy=name&pageNumber=%d&pageSize=%d", organizationID, pageNumber, pageSize)
	request, err := client.newRequestV22(requestURI, http.MethodGet, nil)
	if err != nil {
		return nil, err
	}

	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	if statusCode != http.StatusOK {
		var apiResponse *APIResponseV2

		apiResponse, err = readAPIResponseAsJSON(responseBody, statusCode)
		if err != nil {
			return nil, err
		}

		return nil, apiResponse.ToError("Request to list tag keys failed with status code %d (%s): %s", statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	tagKeys = &TagKeys{}
	err = json.Unmarshal(responseBody, tagKeys)

	return tagKeys, err
}

// CreateTagKey creates a new tag key.
func (client *Client) CreateTagKey(name string, description string, isValueRequired bool, displayOnReports bool) (tagKeyID string, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return "", err
	}

	requestURI := fmt.Sprintf("%s/tag/createTagKey", organizationID)
	request, err := client.newRequestV22(requestURI, http.MethodPost, &tagKey{
		Name:             name,
		Description:      description,
		IsValueRequired:  isValueRequired,
		DisplayOnReports: displayOnReports,
	})
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return "", err
	}

	apiResponse, err := readAPIResponseAsJSON(responseBody, statusCode)
	if err != nil {
		return "", err
	}

	if apiResponse.ResponseCode != ResponseCodeOK {
		return "", apiResponse.ToError("Request to create tag key '%s' failed with unexpected status code %d (%s): %s", name, statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	// Expected: "info" { "name": "tagKeyId", "value": "the-Id-of-the-new-tag-key" }
	if len(apiResponse.FieldMessages) != 1 || apiResponse.FieldMessages[0].FieldName != "tagKeyId" {
		return "", apiResponse.ToError("Received an unexpected response (missing 'tagKeyId') with status code %d (%s): %s", statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	return apiResponse.FieldMessages[0].Message, nil
}

// DeleteTagKey deletes the specified TagKey rule.
func (client *Client) DeleteTagKey(id string) error {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return err
	}

	requestURI := fmt.Sprintf("%s/tag/deleteTagKey", organizationID)
	request, err := client.newRequestV22(requestURI, http.MethodPost,
		&deleteTagKey{id},
	)
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return err
	}

	apiResponse, err := readAPIResponseAsJSON(responseBody, statusCode)
	if err != nil {
		return err
	}

	if apiResponse.ResponseCode != ResponseCodeOK {
		return apiResponse.ToError("Request to delete tag key '%s' failed with unexpected status code %d (%s): %s", id, statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	return nil
}
<|endoftext|>


<commit_before>package eventer

import (
	"github.com/appscode/go/log"
	core "k8s.io/api/core/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/tools/record"
)

const (
	EventReasonCreating                string = "Creating"
	EventReasonPausing                 string = "Pausing"
	EventReasonWipingOut               string = "WipingOut"
	EventReasonFailedToCreate          string = "Failed"
	EventReasonFailedToPause           string = "Failed"
	EventReasonFailedToDelete          string = "Failed"
	EventReasonFailedToWipeOut         string = "Failed"
	EventReasonFailedToGet             string = "Failed"
	EventReasonFailedToInitialize      string = "Failed"
	EventReasonFailedToList            string = "Failed"
	EventReasonFailedToResume          string = "Failed"
	EventReasonFailedToSchedule        string = "Failed"
	EventReasonFailedToStart           string = "Failed"
	EventReasonFailedToUpdate          string = "Failed"
	EventReasonFailedToAddMonitor      string = "Failed"
	EventReasonFailedToDeleteMonitor   string = "Failed"
	EventReasonFailedToUpdateMonitor   string = "Failed"
	EventReasonIgnoredSnapshot         string = "IgnoredSnapshot"
	EventReasonInitializing            string = "Initializing"
	EventReasonInvalid                 string = "Invalid"
	EventReasonInvalidUpdate           string = "InvalidUpdate"
	EventReasonResuming                string = "Resuming"
	EventReasonSnapshotFailed          string = "SnapshotFailed"
	EventReasonStarting                string = "Starting"
	EventReasonSuccessfulCreate        string = "SuccessfulCreate"
	EventReasonSuccessfulPause         string = "SuccessfulPause"
	EventReasonSuccessfulMonitorAdd    string = "SuccessfulMonitorAdd"
	EventReasonSuccessfulMonitorDelete string = "SuccessfulMonitorDelete"
	EventReasonSuccessfulMonitorUpdate string = "SuccessfulMonitorUpdate"
	EventReasonSuccessfulResume        string = "SuccessfulResume"
	EventReasonSuccessfulWipeOut       string = "SuccessfulWipeOut"
	EventReasonSuccessfulSnapshot      string = "SuccessfulSnapshot"
	EventReasonSuccessfulValidate      string = "SuccessfulValidate"
	EventReasonSuccessfulInitialize    string = "SuccessfulInitialize"
)

func NewEventRecorder(client kubernetes.Interface, component string) record.EventRecorder {
	// Event Broadcaster
	broadcaster := record.NewBroadcaster()
	broadcaster.StartEventWatcher(
		func(event *core.Event) {
			if _, err := client.Core().Events(event.Namespace).Create(event); err != nil {
				log.Errorln(err)
			}
		},
	)

	return broadcaster.NewRecorder(scheme.Scheme, core.EventSource{Component: component})
}
<commit_msg>Add Patch Event-Type (#168)<commit_after>package eventer

import (
	"github.com/appscode/go/log"
	core "k8s.io/api/core/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/tools/record"
)

const (
	EventReasonCreating                string = "Creating"
	EventReasonPausing                 string = "Pausing"
	EventReasonWipingOut               string = "WipingOut"
	EventReasonFailedToCreate          string = "Failed"
	EventReasonFailedToPause           string = "Failed"
	EventReasonFailedToDelete          string = "Failed"
	EventReasonFailedToWipeOut         string = "Failed"
	EventReasonFailedToGet             string = "Failed"
	EventReasonFailedToInitialize      string = "Failed"
	EventReasonFailedToList            string = "Failed"
	EventReasonFailedToResume          string = "Failed"
	EventReasonFailedToSchedule        string = "Failed"
	EventReasonFailedToStart           string = "Failed"
	EventReasonFailedToUpdate          string = "Failed"
	EventReasonFailedToAddMonitor      string = "Failed"
	EventReasonFailedToDeleteMonitor   string = "Failed"
	EventReasonFailedToUpdateMonitor   string = "Failed"
	EventReasonIgnoredSnapshot         string = "IgnoredSnapshot"
	EventReasonInitializing            string = "Initializing"
	EventReasonInvalid                 string = "Invalid"
	EventReasonInvalidUpdate           string = "InvalidUpdate"
	EventReasonResuming                string = "Resuming"
	EventReasonSnapshotFailed          string = "SnapshotFailed"
	EventReasonStarting                string = "Starting"
	EventReasonSuccessful              string = "Successful"
	EventReasonSuccessfulCreate        string = "SuccessfulCreate"
	EventReasonSuccessfulPause         string = "SuccessfulPause"
	EventReasonSuccessfulMonitorAdd    string = "SuccessfulMonitorAdd"
	EventReasonSuccessfulMonitorDelete string = "SuccessfulMonitorDelete"
	EventReasonSuccessfulMonitorUpdate string = "SuccessfulMonitorUpdate"
	EventReasonSuccessfulResume        string = "SuccessfulResume"
	EventReasonSuccessfulWipeOut       string = "SuccessfulWipeOut"
	EventReasonSuccessfulSnapshot      string = "SuccessfulSnapshot"
	EventReasonSuccessfulValidate      string = "SuccessfulValidate"
	EventReasonSuccessfulInitialize    string = "SuccessfulInitialize"
)

func NewEventRecorder(client kubernetes.Interface, component string) record.EventRecorder {
	// Event Broadcaster
	broadcaster := record.NewBroadcaster()
	broadcaster.StartEventWatcher(
		func(event *core.Event) {
			if _, err := client.Core().Events(event.Namespace).Create(event); err != nil {
				log.Errorln(err)
			}
		},
	)

	return broadcaster.NewRecorder(scheme.Scheme, core.EventSource{Component: component})
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sysctl

import (
	"fmt"

	v1helper "k8s.io/kubernetes/pkg/api/v1/helper"
	"k8s.io/kubernetes/pkg/kubelet/container"
	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
)

const (
	UnsupportedReason = "SysctlUnsupported"
	// CRI uses semver-compatible API version, while docker does not
	// (e.g., 1.24). Append the version with a ".0".
	dockerMinimumAPIVersion = "1.24.0"

	dockerTypeName = "docker"
)

// TODO: The admission logic in this file is runtime-dependent. It should be
// changed to be generic and CRI-compatible.

type runtimeAdmitHandler struct {
	result lifecycle.PodAdmitResult
}

var _ lifecycle.PodAdmitHandler = &runtimeAdmitHandler{}

// NewRuntimeAdmitHandler returns a sysctlRuntimeAdmitHandler which checks whether
// the given runtime support sysctls.
func NewRuntimeAdmitHandler(runtime container.Runtime) (*runtimeAdmitHandler, error) {
	if runtime.Type() == dockerTypeName {
		v, err := runtime.APIVersion()
		if err != nil {
			return nil, fmt.Errorf("failed to get runtime version: %v", err)
		}

		// only Docker >= 1.12 supports sysctls
		c, err := v.Compare(dockerMinimumAPIVersion)
		if err != nil {
			return nil, fmt.Errorf("failed to compare Docker version for sysctl support: %v", err)
		}
		if c >= 0 {
			return &runtimeAdmitHandler{
				result: lifecycle.PodAdmitResult{
					Admit: true,
				},
			}, nil
		}
		return &runtimeAdmitHandler{
			result: lifecycle.PodAdmitResult{
				Admit:   false,
				Reason:  UnsupportedReason,
				Message: "Docker before 1.12 does not support sysctls",
			},
		}, nil
	}

	// for other runtimes like rkt sysctls are not supported
	return &runtimeAdmitHandler{
		result: lifecycle.PodAdmitResult{
			Admit:   false,
			Reason:  UnsupportedReason,
			Message: fmt.Sprintf("runtime %v does not support sysctls", runtime.Type()),
		},
	}, nil
}

// Admit checks whether the runtime supports sysctls.
func (w *runtimeAdmitHandler) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAdmitResult {
	sysctls, unsafeSysctls, err := v1helper.SysctlsFromPodAnnotations(attrs.Pod.Annotations)
	if err != nil {
		return lifecycle.PodAdmitResult{
			Admit:   false,
			Reason:  AnnotationInvalidReason,
			Message: fmt.Sprintf("invalid sysctl annotation: %v", err),
		}
	}

	if len(sysctls)+len(unsafeSysctls) > 0 {
		return w.result
	}

	return lifecycle.PodAdmitResult{
		Admit: true,
	}
}
<commit_msg>Admit sysctls for other runtime.<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sysctl

import (
	"fmt"

	v1helper "k8s.io/kubernetes/pkg/api/v1/helper"
	"k8s.io/kubernetes/pkg/kubelet/container"
	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
)

const (
	UnsupportedReason = "SysctlUnsupported"
	// CRI uses semver-compatible API version, while docker does not
	// (e.g., 1.24). Append the version with a ".0".
	dockerMinimumAPIVersion = "1.24.0"

	dockerTypeName = "docker"
	rktTypeName    = "rkt"
)

// TODO: The admission logic in this file is runtime-dependent. It should be
// changed to be generic and CRI-compatible.

type runtimeAdmitHandler struct {
	result lifecycle.PodAdmitResult
}

var _ lifecycle.PodAdmitHandler = &runtimeAdmitHandler{}

// NewRuntimeAdmitHandler returns a sysctlRuntimeAdmitHandler which checks whether
// the given runtime support sysctls.
func NewRuntimeAdmitHandler(runtime container.Runtime) (*runtimeAdmitHandler, error) {
	switch runtime.Type() {
	case dockerTypeName:
		v, err := runtime.APIVersion()
		if err != nil {
			return nil, fmt.Errorf("failed to get runtime version: %v", err)
		}

		// only Docker >= 1.12 supports sysctls
		c, err := v.Compare(dockerMinimumAPIVersion)
		if err != nil {
			return nil, fmt.Errorf("failed to compare Docker version for sysctl support: %v", err)
		}
		if c >= 0 {
			return &runtimeAdmitHandler{
				result: lifecycle.PodAdmitResult{
					Admit: true,
				},
			}, nil
		}
		return &runtimeAdmitHandler{
			result: lifecycle.PodAdmitResult{
				Admit:   false,
				Reason:  UnsupportedReason,
				Message: "Docker before 1.12 does not support sysctls",
			},
		}, nil
	case rktTypeName:
		return &runtimeAdmitHandler{
			result: lifecycle.PodAdmitResult{
				Admit:   false,
				Reason:  UnsupportedReason,
				Message: "Rkt does not support sysctls",
			},
		}, nil
	default:
		// Return admit for other runtimes.
		return &runtimeAdmitHandler{
			result: lifecycle.PodAdmitResult{
				Admit: true,
			},
		}, nil
	}
}

// Admit checks whether the runtime supports sysctls.
func (w *runtimeAdmitHandler) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAdmitResult {
	sysctls, unsafeSysctls, err := v1helper.SysctlsFromPodAnnotations(attrs.Pod.Annotations)
	if err != nil {
		return lifecycle.PodAdmitResult{
			Admit:   false,
			Reason:  AnnotationInvalidReason,
			Message: fmt.Sprintf("invalid sysctl annotation: %v", err),
		}
	}

	if len(sysctls)+len(unsafeSysctls) > 0 {
		return w.result
	}

	return lifecycle.PodAdmitResult{
		Admit: true,
	}
}
<|endoftext|>