<commit_before>package main

import (
	"context"
	"net"
	"net/http"
	"runtime"
	"sync"
	"time"

	"github.com/lxc/lxd/lxd/db"
	dbCluster "github.com/lxc/lxd/lxd/db/cluster"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/metrics"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/shared/logger"
)

type metricsCacheEntry struct {
	metrics *metrics.MetricSet
	expiry  time.Time
}

var metricsCache map[string]metricsCacheEntry
var metricsCacheLock sync.Mutex
var metricsLock sync.Mutex

var metricsCmd = APIEndpoint{
	Path: "metrics",

	Get: APIEndpointAction{Handler: metricsGet, AccessHandler: allowMetrics, AllowUntrusted: true},
}

func allowMetrics(d *Daemon, r *http.Request) response.Response {
	// Check if API is wide open.
	if !d.State().GlobalConfig.MetricsAuthentication() {
		return response.EmptySyncResponse
	}

	// If not wide open, apply project access restrictions.
	return allowProjectPermission("containers", "view")(d, r)
}

// swagger:operation GET /1.0/metrics metrics metrics_get
//
// Get metrics
//
// Gets metrics of instances.
//
// ---
// produces:
//   - text/plain
// parameters:
//   - in: query
//     name: project
//     description: Project name
//     type: string
//     example: default
//   - in: query
//     name: target
//     description: Cluster member name
//     type: string
//     example: lxd01
// responses:
//   "200":
//     description: Metrics
//     schema:
//       type: string
//       description: Instance metrics
//   "403":
//     $ref: "#/responses/Forbidden"
//   "500":
//     $ref: "#/responses/InternalServerError"
func metricsGet(d *Daemon, r *http.Request) response.Response {
	projectName := queryParam(r, "project")

	// Forward if requested.
	resp := forwardedResponseIfTargetIsRemote(d, r)
	if resp != nil {
		return resp
	}

	// Wait until daemon is fully started.
	<-d.waitReady.Done()

	// Figure out the projects to retrieve.
	var projectNames []string

	if projectName != "" {
		projectNames = []string{projectName}
	} else {
		// Get all projects.
		err := d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
			projects, err := dbCluster.GetProjects(ctx, tx.Tx())
			if err != nil {
				return err
			}

			for _, project := range projects {
				projectNames = append(projectNames, project.Name)
			}

			return nil
		})
		if err != nil {
			return response.SmartError(err)
		}
	}

	// Prepare response.
	metricSet := metrics.NewMetricSet(nil)

	// Add internal metrics.
	metricSet.Merge(internalMetrics(d))

	// Review the cache.
	metricsCacheLock.Lock()
	projectMissing := []string{}
	for _, project := range projectNames {
		cache, ok := metricsCache[project]
		if !ok || cache.expiry.Before(time.Now()) {
			// If missing or expired, record it.
			projectMissing = append(projectMissing, project)
			continue
		}

		// If present and valid, merge the existing data.
		metricSet.Merge(cache.metrics)
	}

	metricsCacheLock.Unlock()

	// If all valid, return immediately.
	if len(projectMissing) == 0 {
		return response.SyncResponsePlain(true, metricSet.String())
	}

	// Acquire update lock.
	metricsLock.Lock()
	defer metricsLock.Unlock()

	// Check if any of the missing data has been filled in.
	metricsCacheLock.Lock()
	toFetch := []string{}
	for _, project := range projectMissing {
		cache, ok := metricsCache[project]
		if !ok || cache.expiry.Before(time.Now()) {
			// Still missing, queue a re-fetch.
			toFetch = append(toFetch, project)
			continue
		}

		// If present and valid, merge the existing data.
		metricSet.Merge(cache.metrics)
	}

	metricsCacheLock.Unlock()

	// If all valid, return immediately.
	if len(toFetch) == 0 {
		return response.SyncResponsePlain(true, metricSet.String())
	}

	hostInterfaces, _ := net.Interfaces()

	// Prepare temporary metrics storage.
	newMetrics := map[string]*metrics.MetricSet{}
	newMetricsLock := sync.Mutex{}

	// Fetch what's missing.
	wgInstances := sync.WaitGroup{}
	for _, project := range toFetch {
		newMetrics[project] = metrics.NewMetricSet(nil)

		// Get the instances.
		instances, err := instanceLoadNodeProjectAll(d.State(), project, instancetype.Any)
		if err != nil {
			return response.SmartError(err)
		}

		for _, inst := range instances {
			// Ignore stopped instances.
			if !inst.IsRunning() {
				continue
			}

			wgInstances.Add(1)
			go func(inst instance.Instance) {
				defer wgInstances.Done()

				instanceMetrics, err := inst.Metrics(hostInterfaces)
				if err != nil {
					logger.Warn("Failed to get instance metrics", logger.Ctx{"instance": inst.Name(), "project": inst.Project().Name, "err": err})
					return
				}

				// Add the metrics.
				newMetricsLock.Lock()
				defer newMetricsLock.Unlock()

				newMetrics[inst.Project().Name].Merge(instanceMetrics)
			}(inst)
		}
	}

	wgInstances.Wait()

	// Put the new data in the global cache and in response.
	metricsCacheLock.Lock()

	if metricsCache == nil {
		metricsCache = map[string]metricsCacheEntry{}
	}

	for project, entries := range newMetrics {
		metricsCache[project] = metricsCacheEntry{
			expiry:  time.Now().Add(8 * time.Second),
			metrics: entries,
		}

		metricSet.Merge(entries)
	}

	metricsCacheLock.Unlock()

	return response.SyncResponsePlain(true, metricSet.String())
}

func internalMetrics(d *Daemon) *metrics.MetricSet {
	out := metrics.NewMetricSet(nil)

	_ = d.db.Cluster.Transaction(d.shutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error {
		warnings, err := dbCluster.GetWarnings(ctx, tx.Tx())
		if err != nil {
			logger.Warn("Failed to get warnings", logger.Ctx{"err": err})
		} else {
			// Total number of warnings
			out.AddSamples(metrics.WarningsTotal, metrics.Sample{Value: float64(len(warnings))})
		}

		operations, err := dbCluster.GetOperations(ctx, tx.Tx())
		if err != nil {
			logger.Warn("Failed to get operations", logger.Ctx{"err": err})
		} else {
			// Total number of operations
			out.AddSamples(metrics.OperationsTotal, metrics.Sample{Value: float64(len(operations))})
		}

		return nil
	})

	// Daemon uptime
	out.AddSamples(metrics.UptimeSeconds, metrics.Sample{Value: time.Since(d.startTime).Seconds()})

	// Number of goroutines
	out.AddSamples(metrics.GoGoroutines, metrics.Sample{Value: float64(runtime.NumGoroutine())})

	// Go memory stats
	var ms runtime.MemStats

	runtime.ReadMemStats(&ms)

	out.AddSamples(metrics.GoAllocBytes, metrics.Sample{Value: float64(ms.Alloc)})
	out.AddSamples(metrics.GoAllocBytesTotal, metrics.Sample{Value: float64(ms.TotalAlloc)})
	out.AddSamples(metrics.GoBuckHashSysBytes, metrics.Sample{Value: float64(ms.BuckHashSys)})
	out.AddSamples(metrics.GoFreesTotal, metrics.Sample{Value: float64(ms.Frees)})
	out.AddSamples(metrics.GoGCSysBytes, metrics.Sample{Value: float64(ms.GCSys)})
	out.AddSamples(metrics.GoHeapAllocBytes, metrics.Sample{Value: float64(ms.HeapAlloc)})
	out.AddSamples(metrics.GoHeapIdleBytes, metrics.Sample{Value: float64(ms.HeapIdle)})
	out.AddSamples(metrics.GoHeapInuseBytes, metrics.Sample{Value: float64(ms.HeapInuse)})
	out.AddSamples(metrics.GoHeapObjects, metrics.Sample{Value: float64(ms.HeapObjects)})
	out.AddSamples(metrics.GoHeapReleasedBytes, metrics.Sample{Value: float64(ms.HeapReleased)})
	out.AddSamples(metrics.GoHeapSysBytes, metrics.Sample{Value: float64(ms.HeapSys)})
	out.AddSamples(metrics.GoLookupsTotal, metrics.Sample{Value: float64(ms.Lookups)})
	out.AddSamples(metrics.GoMallocsTotal, metrics.Sample{Value: float64(ms.Mallocs)})
	out.AddSamples(metrics.GoMCacheInuseBytes, metrics.Sample{Value: float64(ms.MCacheInuse)})
	out.AddSamples(metrics.GoMCacheSysBytes, metrics.Sample{Value: float64(ms.MCacheSys)})
	out.AddSamples(metrics.GoMSpanInuseBytes, metrics.Sample{Value: float64(ms.MSpanInuse)})
	out.AddSamples(metrics.GoMSpanSysBytes, metrics.Sample{Value: float64(ms.MSpanSys)})
	out.AddSamples(metrics.GoNextGCBytes, metrics.Sample{Value: float64(ms.NextGC)})
	out.AddSamples(metrics.GoOtherSysBytes, metrics.Sample{Value: float64(ms.OtherSys)})
	out.AddSamples(metrics.GoStackInuseBytes, metrics.Sample{Value: float64(ms.StackInuse)})
	out.AddSamples(metrics.GoStackSysBytes, metrics.Sample{Value: float64(ms.StackSys)})
	out.AddSamples(metrics.GoSysBytes, metrics.Sample{Value: float64(ms.Sys)})

	return out
}
<commit_msg>lxd/api/metrics: Fix race condition on map access in metricsGet<commit_after>package main

import (
	"context"
	"net"
	"net/http"
	"runtime"
	"sync"
	"time"

	"github.com/lxc/lxd/lxd/db"
	dbCluster "github.com/lxc/lxd/lxd/db/cluster"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/metrics"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/shared/logger"
)

type metricsCacheEntry struct {
	metrics *metrics.MetricSet
	expiry  time.Time
}

var metricsCache map[string]metricsCacheEntry
var metricsCacheLock sync.Mutex
var metricsLock sync.Mutex

var metricsCmd = APIEndpoint{
	Path: "metrics",

	Get: APIEndpointAction{Handler: metricsGet, AccessHandler: allowMetrics, AllowUntrusted: true},
}

func allowMetrics(d *Daemon, r *http.Request) response.Response {
	// Check if API is wide open.
	if !d.State().GlobalConfig.MetricsAuthentication() {
		return response.EmptySyncResponse
	}

	// If not wide open, apply project access restrictions.
	return allowProjectPermission("containers", "view")(d, r)
}

// swagger:operation GET /1.0/metrics metrics metrics_get
//
// Get metrics
//
// Gets metrics of instances.
//
// ---
// produces:
//   - text/plain
// parameters:
//   - in: query
//     name: project
//     description: Project name
//     type: string
//     example: default
//   - in: query
//     name: target
//     description: Cluster member name
//     type: string
//     example: lxd01
// responses:
//   "200":
//     description: Metrics
//     schema:
//       type: string
//       description: Instance metrics
//   "403":
//     $ref: "#/responses/Forbidden"
//   "500":
//     $ref: "#/responses/InternalServerError"
func metricsGet(d *Daemon, r *http.Request) response.Response {
	projectName := queryParam(r, "project")

	// Forward if requested.
	resp := forwardedResponseIfTargetIsRemote(d, r)
	if resp != nil {
		return resp
	}

	// Wait until daemon is fully started.
	<-d.waitReady.Done()

	// Figure out the projects to retrieve.
	var projectNames []string

	if projectName != "" {
		projectNames = []string{projectName}
	} else {
		// Get all projects.
		err := d.db.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
			projects, err := dbCluster.GetProjects(ctx, tx.Tx())
			if err != nil {
				return err
			}

			for _, project := range projects {
				projectNames = append(projectNames, project.Name)
			}

			return nil
		})
		if err != nil {
			return response.SmartError(err)
		}
	}

	// Prepare response.
	metricSet := metrics.NewMetricSet(nil)

	// Add internal metrics.
	metricSet.Merge(internalMetrics(d))

	// Review the cache.
	metricsCacheLock.Lock()
	projectMissing := []string{}
	for _, project := range projectNames {
		cache, ok := metricsCache[project]
		if !ok || cache.expiry.Before(time.Now()) {
			// If missing or expired, record it.
			projectMissing = append(projectMissing, project)
			continue
		}

		// If present and valid, merge the existing data.
		metricSet.Merge(cache.metrics)
	}

	metricsCacheLock.Unlock()

	// If all valid, return immediately.
	if len(projectMissing) == 0 {
		return response.SyncResponsePlain(true, metricSet.String())
	}

	// Acquire update lock.
	metricsLock.Lock()
	defer metricsLock.Unlock()

	// Check if any of the missing data has been filled in.
	metricsCacheLock.Lock()
	toFetch := []string{}
	for _, project := range projectMissing {
		cache, ok := metricsCache[project]
		if !ok || cache.expiry.Before(time.Now()) {
			// Still missing, queue a re-fetch.
			toFetch = append(toFetch, project)
			continue
		}

		// If present and valid, merge the existing data.
		metricSet.Merge(cache.metrics)
	}

	metricsCacheLock.Unlock()

	// If all valid, return immediately.
	if len(toFetch) == 0 {
		return response.SyncResponsePlain(true, metricSet.String())
	}

	hostInterfaces, _ := net.Interfaces()

	// Prepare temporary metrics storage.
	newMetrics := map[string]*metrics.MetricSet{}
	newMetricsLock := sync.Mutex{}

	// Fetch what's missing.
	wgInstances := sync.WaitGroup{}
	for _, project := range toFetch {
		// Get the instances.
		instances, err := instanceLoadNodeProjectAll(d.State(), project, instancetype.Any)
		if err != nil {
			return response.SmartError(err)
		}

		for _, inst := range instances {
			// Ignore stopped instances.
			if !inst.IsRunning() {
				continue
			}

			wgInstances.Add(1)
			go func(inst instance.Instance) {
				defer wgInstances.Done()

				projectName := inst.Project().Name
				instanceMetrics, err := inst.Metrics(hostInterfaces)
				if err != nil {
					logger.Warn("Failed to get instance metrics", logger.Ctx{"instance": inst.Name(), "project": projectName, "err": err})
					return
				}

				// Add the metrics.
				newMetricsLock.Lock()
				defer newMetricsLock.Unlock()

				// Initialise metrics set for project if needed.
				if newMetrics[projectName] == nil {
					newMetrics[projectName] = metrics.NewMetricSet(nil)
				}

				newMetrics[projectName].Merge(instanceMetrics)
			}(inst)
		}
	}

	wgInstances.Wait()

	// Put the new data in the global cache and in response.
	metricsCacheLock.Lock()

	if metricsCache == nil {
		metricsCache = map[string]metricsCacheEntry{}
	}

	for project, entries := range newMetrics {
		metricsCache[project] = metricsCacheEntry{
			expiry:  time.Now().Add(8 * time.Second),
			metrics: entries,
		}

		metricSet.Merge(entries)
	}

	metricsCacheLock.Unlock()

	return response.SyncResponsePlain(true, metricSet.String())
}

func internalMetrics(d *Daemon) *metrics.MetricSet {
	out := metrics.NewMetricSet(nil)

	_ = d.db.Cluster.Transaction(d.shutdownCtx, func(ctx context.Context, tx *db.ClusterTx) error {
		warnings, err := dbCluster.GetWarnings(ctx, tx.Tx())
		if err != nil {
			logger.Warn("Failed to get warnings", logger.Ctx{"err": err})
		} else {
			// Total number of warnings
			out.AddSamples(metrics.WarningsTotal, metrics.Sample{Value: float64(len(warnings))})
		}

		operations, err := dbCluster.GetOperations(ctx, tx.Tx())
		if err != nil {
			logger.Warn("Failed to get operations", logger.Ctx{"err": err})
		} else {
			// Total number of operations
			out.AddSamples(metrics.OperationsTotal, metrics.Sample{Value: float64(len(operations))})
		}

		return nil
	})

	// Daemon uptime
	out.AddSamples(metrics.UptimeSeconds, metrics.Sample{Value: time.Since(d.startTime).Seconds()})

	// Number of goroutines
	out.AddSamples(metrics.GoGoroutines, metrics.Sample{Value: float64(runtime.NumGoroutine())})

	// Go memory stats
	var ms runtime.MemStats

	runtime.ReadMemStats(&ms)

	out.AddSamples(metrics.GoAllocBytes, metrics.Sample{Value: float64(ms.Alloc)})
	out.AddSamples(metrics.GoAllocBytesTotal, metrics.Sample{Value: float64(ms.TotalAlloc)})
	out.AddSamples(metrics.GoBuckHashSysBytes, metrics.Sample{Value: float64(ms.BuckHashSys)})
	out.AddSamples(metrics.GoFreesTotal, metrics.Sample{Value: float64(ms.Frees)})
	out.AddSamples(metrics.GoGCSysBytes, metrics.Sample{Value: float64(ms.GCSys)})
	out.AddSamples(metrics.GoHeapAllocBytes, metrics.Sample{Value: float64(ms.HeapAlloc)})
	out.AddSamples(metrics.GoHeapIdleBytes, metrics.Sample{Value: float64(ms.HeapIdle)})
	out.AddSamples(metrics.GoHeapInuseBytes, metrics.Sample{Value: float64(ms.HeapInuse)})
	out.AddSamples(metrics.GoHeapObjects, metrics.Sample{Value: float64(ms.HeapObjects)})
	out.AddSamples(metrics.GoHeapReleasedBytes, metrics.Sample{Value: float64(ms.HeapReleased)})
	out.AddSamples(metrics.GoHeapSysBytes, metrics.Sample{Value: float64(ms.HeapSys)})
	out.AddSamples(metrics.GoLookupsTotal, metrics.Sample{Value: float64(ms.Lookups)})
	out.AddSamples(metrics.GoMallocsTotal, metrics.Sample{Value: float64(ms.Mallocs)})
	out.AddSamples(metrics.GoMCacheInuseBytes, metrics.Sample{Value: float64(ms.MCacheInuse)})
	out.AddSamples(metrics.GoMCacheSysBytes, metrics.Sample{Value: float64(ms.MCacheSys)})
	out.AddSamples(metrics.GoMSpanInuseBytes, metrics.Sample{Value: float64(ms.MSpanInuse)})
	out.AddSamples(metrics.GoMSpanSysBytes, metrics.Sample{Value: float64(ms.MSpanSys)})
	out.AddSamples(metrics.GoNextGCBytes, metrics.Sample{Value: float64(ms.NextGC)})
	out.AddSamples(metrics.GoOtherSysBytes, metrics.Sample{Value: float64(ms.OtherSys)})
	out.AddSamples(metrics.GoStackInuseBytes, metrics.Sample{Value: float64(ms.StackInuse)})
	out.AddSamples(metrics.GoStackSysBytes, metrics.Sample{Value: float64(ms.StackSys)})
	out.AddSamples(metrics.GoSysBytes, metrics.Sample{Value: float64(ms.Sys)})

	return out
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/gorilla/mux"
	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/types"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/version"
)

var projectsCmd = Command{
	name: "projects",
	get:  apiProjectsGet,
	post: apiProjectsPost,
}

var projectCmd = Command{
	name:   "projects/{name}",
	get:    apiProjectGet,
	post:   apiProjectPost,
	put:    apiProjectPut,
	patch:  apiProjectPatch,
	delete: apiProjectDelete,
}

func apiProjectsGet(d *Daemon, r *http.Request) Response {
	recursion := util.IsRecursionRequest(r)

	var result interface{}
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		filter := db.ProjectFilter{}
		if recursion {
			result, err = tx.ProjectList(filter)
		} else {
			result, err = tx.ProjectURIs(filter)
		}
		return err
	})
	if err != nil {
		return SmartError(err)
	}

	return SyncResponse(true, result)
}

func apiProjectsPost(d *Daemon, r *http.Request) Response {
	// Parse the request
	project := api.ProjectsPost{}

	// Set default features
	if project.Config == nil {
		project.Config = map[string]string{}
	}
	for _, feature := range []string{"features.images", "features.profiles"} {
		_, ok := project.Config[feature]
		if !ok {
			project.Config[feature] = "true"
		}
	}

	err := json.NewDecoder(r.Body).Decode(&project)
	if err != nil {
		return BadRequest(err)
	}

	// Sanity checks
	if project.Name == "" {
		return BadRequest(fmt.Errorf("No name provided"))
	}

	if strings.Contains(project.Name, "/") {
		return BadRequest(fmt.Errorf("Project names may not contain slashes"))
	}

	if project.Name == "*" {
		return BadRequest(fmt.Errorf("Reserved project name"))
	}

	if shared.StringInSlice(project.Name, []string{".", ".."}) {
		return BadRequest(fmt.Errorf("Invalid project name '%s'", project.Name))
	}

	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		_, err := tx.ProjectCreate(project)
		if err != nil {
			return errors.Wrap(err, "Add project to database")
		}

		if project.Config["features.profiles"] == "true" {
			err = apiProjectCreateDefaultProfile(tx, project.Name)
			if err != nil {
				return err
			}
		}

		return nil
	})
	if err != nil {
		return SmartError(fmt.Errorf("Error inserting %s into database: %s", project.Name, err))
	}

	return SyncResponseLocation(true, nil, fmt.Sprintf("/%s/projects/%s", version.APIVersion, project.Name))
}

// Create the default profile of a project.
func apiProjectCreateDefaultProfile(tx *db.ClusterTx, project string) error {
	// Create a default profile
	profile := db.Profile{}
	profile.Project = project
	profile.Name = "default"
	profile.Description = fmt.Sprintf("Default LXD profile for project %s", project)
	profile.Config = map[string]string{}
	profile.Devices = types.Devices{}

	_, err := tx.ProfileCreate(profile)
	if err != nil {
		return errors.Wrap(err, "Add default profile to database")
	}
	return nil
}

func apiProjectGet(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	// Get the database entry
	var project *api.Project
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		project, err = tx.ProjectGet(name)
		return err
	})
	if err != nil {
		return SmartError(err)
	}

	etag := []interface{}{
		project.Description,
		project.Config["features.images"],
		project.Config["features.profiles"],
	}

	return SyncResponseETag(true, project, etag)
}

func apiProjectPut(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	// Get the current data
	var project *api.Project
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		project, err = tx.ProjectGet(name)
		return err
	})
	if err != nil {
		return SmartError(err)
	}

	// Validate ETag
	etag := []interface{}{
		project.Description,
		project.Config["features.images"],
		project.Config["features.profiles"],
	}
	err = util.EtagCheck(r, etag)
	if err != nil {
		return PreconditionFailed(err)
	}

	// Parse the request
	req := api.ProjectPut{}

	err = json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		return BadRequest(err)
	}

	return apiProjectChange(d, project, req)
}

func apiProjectPatch(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	// Get the current data
	var project *api.Project
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		project, err = tx.ProjectGet(name)
		return err
	})
	if err != nil {
		return SmartError(err)
	}

	// Validate ETag
	etag := []interface{}{
		project.Description,
		project.Config["features.images"],
		project.Config["features.profiles"],
	}
	err = util.EtagCheck(r, etag)
	if err != nil {
		return PreconditionFailed(err)
	}

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return InternalError(err)
	}

	rdr1 := ioutil.NopCloser(bytes.NewBuffer(body))
	rdr2 := ioutil.NopCloser(bytes.NewBuffer(body))

	reqRaw := shared.Jmap{}
	if err := json.NewDecoder(rdr1).Decode(&reqRaw); err != nil {
		return BadRequest(err)
	}

	req := api.ProjectPut{}
	if err := json.NewDecoder(rdr2).Decode(&req); err != nil {
		return BadRequest(err)
	}

	// Check what was actually set in the query
	_, err = reqRaw.GetString("description")
	if err != nil {
		req.Description = project.Description
	}

	_, err = reqRaw.GetBool("features.images")
	if err != nil {
		req.Config["features.images"] = project.Config["features.images"]
	}

	_, err = reqRaw.GetBool("features.profiles")
	if err != nil {
		req.Config["features.images"] = project.Config["features.profiles"]
	}

	return apiProjectChange(d, project, req)
}

// Common logic between PUT and PATCH.
func apiProjectChange(d *Daemon, project *api.Project, req api.ProjectPut) Response {
	// Flag indicating if any feature has changed.
	featuresChanged := req.Config["features.images"] != project.Config["features.images"] || req.Config["features.profiles"] != project.Config["features.profiles"]

	// Sanity checks
	if project.Name == "default" && featuresChanged {
		return BadRequest(fmt.Errorf("You can't change the features of the default project"))
	}

	if !apiProjectIsEmpty(project) && featuresChanged {
		return BadRequest(fmt.Errorf("Features can only be changed on empty projects"))
	}

	// Update the database entry
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		err := tx.ProjectUpdate(project.Name, req)
		if err != nil {
			return errors.Wrap(err, "Persist profile changes")
		}

		if req.Config["features.profiles"] != project.Config["features.profiles"] {
			if req.Config["features.profiles"] == "true" {
				err = apiProjectCreateDefaultProfile(tx, project.Name)
				if err != nil {
					return err
				}
			} else {
				// Delete the project-specific default profile.
				err = tx.ProfileDelete(project.Name, "default")
				if err != nil {
					return errors.Wrap(err, "Delete project default profile")
				}
			}
		}

		return nil

	})

	if err != nil {
		return SmartError(err)
	}

	return EmptySyncResponse
}

func apiProjectPost(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	// Parse the request
	req := api.ProjectPost{}

	err := json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		return BadRequest(err)
	}

	// Sanity checks
	if name == "default" {
		return Forbidden(fmt.Errorf("The 'default' project cannot be renamed"))
	}

	// Perform the rename
	run := func(op *operation) error {
		err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
			project, err := tx.ProjectGet(req.Name)
			if err != nil && err != db.ErrNoSuchObject {
				return errors.Wrapf(err, "Check if project %q exists", req.Name)
			}

			if project != nil {
				return fmt.Errorf("A project named '%s' already exists", req.Name)
			}

			project, err = tx.ProjectGet(name)
			if err != nil {
				return errors.Wrapf(err, "Fetch project %q", name)
			}

			if !apiProjectIsEmpty(project) {
				return fmt.Errorf("Only empty projects can be renamed")
			}

			return tx.ProjectRename(name, req.Name)
		})

		return err
	}

	op, err := operationCreate(d.cluster, "", operationClassTask, db.OperationProjectRename, nil, nil, run, nil, nil)
	if err != nil {
		return InternalError(err)
	}

	return OperationResponse(op)
}

func apiProjectDelete(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	// Sanity checks
	if name == "default" {
		return Forbidden(fmt.Errorf("The 'default' project cannot be deleted"))
	}

	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		project, err := tx.ProjectGet(name)
		if err != nil {
			return errors.Wrapf(err, "Fetch project %q", name)
		}
		if !apiProjectIsEmpty(project) {
			return fmt.Errorf("Only empty projects can be removed")
		}

		return tx.ProjectDelete(name)
	})

	if err != nil {
		return SmartError(err)
	}

	return EmptySyncResponse
}

// Check if a project is empty.
func apiProjectIsEmpty(project *api.Project) bool {
	if len(project.UsedBy) > 0 {
		// Check if the only entity is the default profile.
		if len(project.UsedBy) == 1 && strings.Contains(project.UsedBy[0], "/profiles/default") {
			return true
		}
		return false
	}
	return true
}

// Add the "<project>_" prefix when the given project name is not "default".
func projectPrefix(project string, s string) string {
	if project != "default" {
		s = fmt.Sprintf("%s_%s", project, s)
	}
	return s
}
<commit_msg>lxd/projects: Add config validation<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/gorilla/mux"
	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/types"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/version"
)

var projectsCmd = Command{
	name: "projects",
	get:  apiProjectsGet,
	post: apiProjectsPost,
}

var projectCmd = Command{
	name:   "projects/{name}",
	get:    apiProjectGet,
	post:   apiProjectPost,
	put:    apiProjectPut,
	patch:  apiProjectPatch,
	delete: apiProjectDelete,
}

func apiProjectsGet(d *Daemon, r *http.Request) Response {
	recursion := util.IsRecursionRequest(r)

	var result interface{}
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		filter := db.ProjectFilter{}
		if recursion {
			result, err = tx.ProjectList(filter)
		} else {
			result, err = tx.ProjectURIs(filter)
		}
		return err
	})
	if err != nil {
		return SmartError(err)
	}

	return SyncResponse(true, result)
}

func apiProjectsPost(d *Daemon, r *http.Request) Response {
	// Parse the request
	project := api.ProjectsPost{}

	// Set default features
	if project.Config == nil {
		project.Config = map[string]string{}
	}
	for _, feature := range []string{"features.images", "features.profiles"} {
		_, ok := project.Config[feature]
		if !ok {
			project.Config[feature] = "true"
		}
	}

	err := json.NewDecoder(r.Body).Decode(&project)
	if err != nil {
		return BadRequest(err)
	}

	// Sanity checks
	if project.Name == "" {
		return BadRequest(fmt.Errorf("No name provided"))
	}

	if strings.Contains(project.Name, "/") {
		return BadRequest(fmt.Errorf("Project names may not contain slashes"))
	}

	if project.Name == "*" {
		return BadRequest(fmt.Errorf("Reserved project name"))
	}

	if shared.StringInSlice(project.Name, []string{".", ".."}) {
		return BadRequest(fmt.Errorf("Invalid project name '%s'", project.Name))
	}

	// Validate the configuration
	err = projectValidateConfig(project.Config)
	if err != nil {
		return BadRequest(err)
	}

	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		_, err := tx.ProjectCreate(project)
		if err != nil {
			return errors.Wrap(err, "Add project to database")
		}

		if project.Config["features.profiles"] == "true" {
			err = apiProjectCreateDefaultProfile(tx, project.Name)
			if err != nil {
				return err
			}
		}

		return nil
	})
	if err != nil {
		return SmartError(fmt.Errorf("Error inserting %s into database: %s", project.Name, err))
	}

	return SyncResponseLocation(true, nil, fmt.Sprintf("/%s/projects/%s", version.APIVersion, project.Name))
}

// Create the default profile of a project.
func apiProjectCreateDefaultProfile(tx *db.ClusterTx, project string) error {
	// Create a default profile
	profile := db.Profile{}
	profile.Project = project
	profile.Name = "default"
	profile.Description = fmt.Sprintf("Default LXD profile for project %s", project)
	profile.Config = map[string]string{}
	profile.Devices = types.Devices{}

	_, err := tx.ProfileCreate(profile)
	if err != nil {
		return errors.Wrap(err, "Add default profile to database")
	}
	return nil
}

func apiProjectGet(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	// Get the database entry
	var project *api.Project
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		project, err = tx.ProjectGet(name)
		return err
	})
	if err != nil {
		return SmartError(err)
	}

	etag := []interface{}{
		project.Description,
		project.Config["features.images"],
		project.Config["features.profiles"],
	}

	return SyncResponseETag(true, project, etag)
}

func apiProjectPut(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	// Get the current data
	var project *api.Project
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		project, err = tx.ProjectGet(name)
		return err
	})
	if err != nil {
		return SmartError(err)
	}

	// Validate ETag
	etag := []interface{}{
		project.Description,
		project.Config["features.images"],
		project.Config["features.profiles"],
	}
	err = util.EtagCheck(r, etag)
	if err != nil {
		return PreconditionFailed(err)
	}

	// Parse the request
	req := api.ProjectPut{}

	err = json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		return BadRequest(err)
	}

	return apiProjectChange(d, project, req)
}

func apiProjectPatch(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	// Get the current data
	var project *api.Project
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		project, err = tx.ProjectGet(name)
		return err
	})
	if err != nil {
		return SmartError(err)
	}

	// Validate ETag
	etag := []interface{}{
		project.Description,
		project.Config["features.images"],
		project.Config["features.profiles"],
	}
	err = util.EtagCheck(r, etag)
	if err != nil {
		return PreconditionFailed(err)
	}

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return InternalError(err)
	}

	rdr1 := ioutil.NopCloser(bytes.NewBuffer(body))
	rdr2 := ioutil.NopCloser(bytes.NewBuffer(body))

	reqRaw := shared.Jmap{}
	if err := json.NewDecoder(rdr1).Decode(&reqRaw); err != nil {
		return BadRequest(err)
	}

	req := api.ProjectPut{}
	if err := json.NewDecoder(rdr2).Decode(&req); err != nil {
		return BadRequest(err)
	}

	// Check what was actually set in the query
	_, err = reqRaw.GetString("description")
	if err != nil {
		req.Description = project.Description
	}

	_, err = reqRaw.GetBool("features.images")
	if err != nil {
		req.Config["features.images"] = project.Config["features.images"]
	}

	_, err = reqRaw.GetBool("features.profiles")
	if err != nil {
		req.Config["features.images"] = project.Config["features.profiles"]
	}

	return apiProjectChange(d, project, req)
}

// Common logic between PUT and PATCH.
func apiProjectChange(d *Daemon, project *api.Project, req api.ProjectPut) Response {
	// Flag indicating if any feature has changed.
	featuresChanged := req.Config["features.images"] != project.Config["features.images"] || req.Config["features.profiles"] != project.Config["features.profiles"]

	// Sanity checks
	if project.Name == "default" && featuresChanged {
		return BadRequest(fmt.Errorf("You can't change the features of the default project"))
	}

	if !apiProjectIsEmpty(project) && featuresChanged {
		return BadRequest(fmt.Errorf("Features can only be changed on empty projects"))
	}

	// Validate the configuration
	err := projectValidateConfig(req.Config)
	if err != nil {
		return BadRequest(err)
	}

	// Update the database entry
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		err := tx.ProjectUpdate(project.Name, req)
		if err != nil {
			return errors.Wrap(err, "Persist profile changes")
		}

		if req.Config["features.profiles"] != project.Config["features.profiles"] {
			if req.Config["features.profiles"] == "true" {
				err = apiProjectCreateDefaultProfile(tx, project.Name)
				if err != nil {
					return err
				}
			} else {
				// Delete the project-specific default profile.
				err = tx.ProfileDelete(project.Name, "default")
				if err != nil {
					return errors.Wrap(err, "Delete project default profile")
				}
			}
		}

		return nil

	})

	if err != nil {
		return SmartError(err)
	}

	return EmptySyncResponse
}

func apiProjectPost(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	// Parse the request
	req := api.ProjectPost{}

	err := json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		return BadRequest(err)
	}

	// Sanity checks
	if name == "default" {
		return Forbidden(fmt.Errorf("The 'default' project cannot be renamed"))
	}

	// Perform the rename
	run := func(op *operation) error {
		err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
			project, err := tx.ProjectGet(req.Name)
			if err != nil && err != db.ErrNoSuchObject {
				return errors.Wrapf(err, "Check if project %q exists", req.Name)
			}

			if project != nil {
				return fmt.Errorf("A project named '%s' already exists", req.Name)
			}

			project, err = tx.ProjectGet(name)
			if err != nil {
				return errors.Wrapf(err, "Fetch project %q", name)
			}

			if !apiProjectIsEmpty(project) {
				return fmt.Errorf("Only empty projects can be renamed")
			}

			return tx.ProjectRename(name, req.Name)
		})

		return err
	}

	op, err := operationCreate(d.cluster, "", operationClassTask, db.OperationProjectRename, nil, nil, run, nil, nil)
	if err != nil {
		return InternalError(err)
	}

	return OperationResponse(op)
}

func apiProjectDelete(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]

	// Sanity checks
	if name == "default" {
		return Forbidden(fmt.Errorf("The 'default' project cannot be deleted"))
	}

	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		project, err := tx.ProjectGet(name)
		if err != nil {
			return errors.Wrapf(err, "Fetch project %q", name)
		}
		if !apiProjectIsEmpty(project) {
			return fmt.Errorf("Only empty projects can be removed")
		}

		return tx.ProjectDelete(name)
	})

	if err != nil {
		return SmartError(err)
	}

	return EmptySyncResponse
}

// Check if a project is empty.
func apiProjectIsEmpty(project *api.Project) bool {
	if len(project.UsedBy) > 0 {
		// Check if the only entity is the default profile.
		if len(project.UsedBy) == 1 && strings.Contains(project.UsedBy[0], "/profiles/default") {
			return true
		}
		return false
	}
	return true
}

// Add the "<project>_" prefix when the given project name is not "default".
func projectPrefix(project string, s string) string {
	if project != "default" {
		s = fmt.Sprintf("%s_%s", project, s)
	}
	return s
}

// Validate the project configuration
var projectConfigKeys = map[string]func(value string) error{
	"features.profiles": shared.IsBool,
	"features.images":   shared.IsBool,
}

func projectValidateConfig(config map[string]string) error {
	for k, v := range config {
		key := k

		// User keys are free for all
		if strings.HasPrefix(key, "user.") {
			continue
		}

		// Then validate
		validator, ok := projectConfigKeys[key]
		if !ok {
			return fmt.Errorf("Invalid project configuration key: %s", k)
		}

		err := validator(v)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package macaroons

import (
	"crypto/rand"
	"fmt"
	"io"

	"github.com/boltdb/bolt"
)

const (
	// RootKeyLen is the length of a root key.
	RootKeyLen = 32
)

var (
	// rootKeyBucketName is the name of the root key store bucket.
	rootKeyBucketName = []byte("macrootkeys")

	// defaultRootKeyID is the ID of the default root key. The first is
	// just 0, to emulate the memory storage that comes with bakery.
	//
	// TODO(aakselrod): Add support for key rotation.
	defaultRootKeyID = "0"

	// macaroonBucketName is the name of the macaroon store bucket.
	macaroonBucketName = []byte("macaroons")
)

// RootKeyStorage implements the bakery.RootKeyStorage interface.
type RootKeyStorage struct {
	*bolt.DB
}

// NewRootKeyStorage creates a RootKeyStorage instance.
// TODO(aakselrod): Add support for encryption of data with passphrase.
func NewRootKeyStorage(db *bolt.DB) (*RootKeyStorage, error) {
	// If the store's bucket doesn't exist, create it.
	err := db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists(rootKeyBucketName)
		return err
	})
	if err != nil {
		return nil, err
	}

	// Return the DB wrapped in a RootKeyStorage object.
	return &RootKeyStorage{db}, nil
}

// Get implements the Get method for the bakery.RootKeyStorage interface.
func (r *RootKeyStorage) Get(id string) ([]byte, error) {
	var rootKey []byte
	err := r.View(func(tx *bolt.Tx) error {
		rootKey = tx.Bucket(rootKeyBucketName).Get([]byte(id))
		if len(rootKey) == 0 {
			return fmt.Errorf("root key with id %s doesn't exist",
				id)
		}
		return nil
	})
	if err != nil {
		return nil, err
	}

	return rootKey, nil
}

// RootKey implements the RootKey method for the bakery.RootKeyStorage
// interface.
// TODO(aakselrod): Add support for key rotation.
func (r *RootKeyStorage) RootKey() ([]byte, string, error) {
	var rootKey []byte
	id := defaultRootKeyID
	err := r.Update(func(tx *bolt.Tx) error {
		ns := tx.Bucket(rootKeyBucketName)
		rootKey = ns.Get([]byte(id))

		// If there's no root key stored in the bucket yet, create one.
		if len(rootKey) != 0 {
			return nil
		}

		// Create a RootKeyLen-byte root key.
		rootKey = make([]byte, RootKeyLen)
		if _, err := io.ReadFull(rand.Reader, rootKey[:]); err != nil {
			return err
		}
		return ns.Put([]byte(id), rootKey)
	})
	if err != nil {
		return nil, "", err
	}

	return rootKey, id, nil
}

// Storage implements the bakery.Storage interface.
type Storage struct {
	*bolt.DB
}

// NewStorage creates a Storage instance.
//
// TODO(aakselrod): Add support for encryption of data with passphrase.
func NewStorage(db *bolt.DB) (*Storage, error) {
	// If the store's bucket doesn't exist, create it.
	err := db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists(macaroonBucketName)
		return err
	})
	if err != nil {
		return nil, err
	}

	// Return the DB wrapped in a Storage object.
	return &Storage{db}, nil
}

// Put implements the Put method for the bakery.Storage interface.
func (s *Storage) Put(location string, item string) error {
	return s.Update(func(tx *bolt.Tx) error {
		return tx.Bucket(macaroonBucketName).Put([]byte(location),
			[]byte(item))
	})
}

// Get implements the Get method for the bakery.Storage interface.
func (s *Storage) Get(location string) (string, error) {
	var item string
	err := s.View(func(tx *bolt.Tx) error {
		itemBytes := tx.Bucket(macaroonBucketName).Get([]byte(location))
		if len(itemBytes) == 0 {
			return fmt.Errorf("couldn't get item for location %s",
				location)
		}
		item = string(itemBytes)
		return nil
	})
	if err != nil {
		return "", err
	}

	return item, nil
}

// Del implements the Del method for the bakery.Storage interface.
func (s *Storage) Del(location string) error {
	return s.Update(func(tx *bolt.Tx) error {
		return tx.Bucket(macaroonBucketName).Delete([]byte(location))
	})
}
<commit_msg>macaroons: ensure all bytes read from db are copied before returning<commit_after>package macaroons

import (
	"crypto/rand"
	"fmt"
	"io"

	"github.com/boltdb/bolt"
)

const (
	// RootKeyLen is the length of a root key.
	RootKeyLen = 32
)

var (
	// rootKeyBucketName is the name of the root key store bucket.
	rootKeyBucketName = []byte("macrootkeys")

	// defaultRootKeyID is the ID of the default root key. The first is
	// just 0, to emulate the memory storage that comes with bakery.
	//
	// TODO(aakselrod): Add support for key rotation.
	defaultRootKeyID = "0"

	// macaroonBucketName is the name of the macaroon store bucket.
	macaroonBucketName = []byte("macaroons")
)

// RootKeyStorage implements the bakery.RootKeyStorage interface.
type RootKeyStorage struct {
	*bolt.DB
}

// NewRootKeyStorage creates a RootKeyStorage instance.
// TODO(aakselrod): Add support for encryption of data with passphrase.
func NewRootKeyStorage(db *bolt.DB) (*RootKeyStorage, error) {
	// If the store's bucket doesn't exist, create it.
	err := db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists(rootKeyBucketName)
		return err
	})
	if err != nil {
		return nil, err
	}

	// Return the DB wrapped in a RootKeyStorage object.
	return &RootKeyStorage{db}, nil
}

// Get implements the Get method for the bakery.RootKeyStorage interface.
func (r *RootKeyStorage) Get(id string) ([]byte, error) {
	var rootKey []byte
	err := r.View(func(tx *bolt.Tx) error {
		dbKey := tx.Bucket(rootKeyBucketName).Get([]byte(id))
		if len(dbKey) == 0 {
			return fmt.Errorf("root key with id %s doesn't exist",
				id)
		}

		rootKey = make([]byte, len(dbKey))
		copy(rootKey[:], dbKey)
		return nil
	})
	if err != nil {
		return nil, err
	}

	return rootKey, nil
}

// RootKey implements the RootKey method for the bakery.RootKeyStorage
// interface.
// TODO(aakselrod): Add support for key rotation.
func (r *RootKeyStorage) RootKey() ([]byte, string, error) {
	var rootKey []byte
	id := defaultRootKeyID
	err := r.Update(func(tx *bolt.Tx) error {
		ns := tx.Bucket(rootKeyBucketName)
		rootKey = ns.Get([]byte(id))

		// If there's no root key stored in the bucket yet, create one.
		if len(rootKey) != 0 {
			return nil
		}

		// Create a RootKeyLen-byte root key.
		rootKey = make([]byte, RootKeyLen)
		if _, err := io.ReadFull(rand.Reader, rootKey[:]); err != nil {
			return err
		}
		return ns.Put([]byte(id), rootKey)
	})
	if err != nil {
		return nil, "", err
	}

	return rootKey, id, nil
}

// Storage implements the bakery.Storage interface.
type Storage struct {
	*bolt.DB
}

// NewStorage creates a Storage instance.
//
// TODO(aakselrod): Add support for encryption of data with passphrase.
func NewStorage(db *bolt.DB) (*Storage, error) {
	// If the store's bucket doesn't exist, create it.
	err := db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists(macaroonBucketName)
		return err
	})
	if err != nil {
		return nil, err
	}

	// Return the DB wrapped in a Storage object.
	return &Storage{db}, nil
}

// Put implements the Put method for the bakery.Storage interface.
func (s *Storage) Put(location string, item string) error {
	return s.Update(func(tx *bolt.Tx) error {
		return tx.Bucket(macaroonBucketName).Put([]byte(location),
			[]byte(item))
	})
}

// Get implements the Get method for the bakery.Storage interface.
func (s *Storage) Get(location string) (string, error) {
	var item []byte
	err := s.View(func(tx *bolt.Tx) error {
		itemBytes := tx.Bucket(macaroonBucketName).Get([]byte(location))
		if len(itemBytes) == 0 {
			return fmt.Errorf("couldn't get item for location %s",
				location)
		}

		item = make([]byte, len(itemBytes))
		copy(item, itemBytes)
		return nil
	})
	if err != nil {
		return "", err
	}

	return string(item), nil
}

// Del implements the Del method for the bakery.Storage interface.
func (s *Storage) Del(location string) error {
	return s.Update(func(tx *bolt.Tx) error {
		return tx.Bucket(macaroonBucketName).Delete([]byte(location))
	})
}
<|endoftext|>


<commit_before>package serf

import (
	"log"
)

// delegate is the memberlist.Delegate implementation that Serf uses.
type delegate struct {
	serf *Serf
}

func (d *delegate) NodeMeta(limit int) []byte {
	roleBytes := []byte(d.serf.config.Role)
	if len(roleBytes) > limit {
		roleBytes = roleBytes[:limit]
	}

	return roleBytes
}

func (d *delegate) NotifyMsg(buf []byte) {
	// If we didn't actually receive any data, then ignore it.
	if len(buf) == 0 {
		return
	}

	rebroadcast := false
	t := messageType(buf[0])
	switch t {
	case messageLeaveType:
		var leave messageLeave
		if err := decodeMessage(buf[1:], &leave); err != nil {
			log.Printf("[ERR] Error decoding leave message: %s", err)
			break
		}

		rebroadcast = d.serf.handleNodeLeaveIntent(&leave)
	case messageRemoveFailedType:
		var remove messageRemoveFailed
		if err := decodeMessage(buf[1:], &remove); err != nil {
			log.Printf("[ERR] Error decoding remove message: %s", err)
			break
		}

		rebroadcast = d.serf.handleNodeForceRemove(&remove)
	default:
		log.Printf("[WARN] Received message of unknown type: %d", t)
	}

	if rebroadcast {
		/*
			TODO(mitchellh): Causes tests to break...
			d.serf.broadcasts.QueueBroadcast(&broadcast{
				key:    key,
				msg:    buf,
				notify: nil,
			})
		*/
	}
}

func (d *delegate) GetBroadcasts(overhead, limit int) [][]byte {
	return d.serf.broadcasts.GetBroadcasts(overhead, limit)
}

func (d *delegate) LocalState() []byte {
	return nil
}

func (d *delegate) MergeRemoteState([]byte) {
}
<commit_msg>serf: remove unused key, even in comment<commit_after>package serf

import (
	"log"
)

// delegate is the memberlist.Delegate implementation that Serf uses.
type delegate struct {
	serf *Serf
}

func (d *delegate) NodeMeta(limit int) []byte {
	roleBytes := []byte(d.serf.config.Role)
	if len(roleBytes) > limit {
		roleBytes = roleBytes[:limit]
	}

	return roleBytes
}

func (d *delegate) NotifyMsg(buf []byte) {
	// If we didn't actually receive any data, then ignore it.
	if len(buf) == 0 {
		return
	}

	rebroadcast := false
	t := messageType(buf[0])
	switch t {
	case messageLeaveType:
		var leave messageLeave
		if err := decodeMessage(buf[1:], &leave); err != nil {
			log.Printf("[ERR] Error decoding leave message: %s", err)
			break
		}

		rebroadcast = d.serf.handleNodeLeaveIntent(&leave)
	case messageRemoveFailedType:
		var remove messageRemoveFailed
		if err := decodeMessage(buf[1:], &remove); err != nil {
			log.Printf("[ERR] Error decoding remove message: %s", err)
			break
		}

		rebroadcast = d.serf.handleNodeForceRemove(&remove)
	default:
		log.Printf("[WARN] Received message of unknown type: %d", t)
	}

	if rebroadcast {
		/*
			TODO(mitchellh): Causes tests to break...
			d.serf.broadcasts.QueueBroadcast(&broadcast{
				msg:    buf,
				notify: nil,
			})
		*/
	}
}

func (d *delegate) GetBroadcasts(overhead, limit int) [][]byte {
	return d.serf.broadcasts.GetBroadcasts(overhead, limit)
}

func (d *delegate) LocalState() []byte {
	return nil
}

func (d *delegate) MergeRemoteState([]byte) {
}
<|endoftext|>


<commit_before>package mackerelplugin

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"math"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/mackerelio/golib/pluginutil"
)

// Metrics represents definition of a metric
type Metrics struct {
	Name         string  `json:"name"`
	Label        string  `json:"label"`
	Diff         bool    `json:"-"`
	Type         string  `json:"-"`
	Stacked      bool    `json:"stacked"`
	Scale        float64 `json:"-"`
	AbsoluteName bool    `json:"-"`
}

// Graphs represents definition of a graph
type Graphs struct {
	Label   string    `json:"label"`
	Unit    string    `json:"unit"`
	Metrics []Metrics `json:"metrics"`
}

// MetricValues represents a collection of metric values and its timestamp
type MetricValues struct {
	Values    map[string]interface{}
	Timestamp time.Time
}

// Plugin is old interface of mackerel-plugin
type Plugin interface {
	FetchMetrics() (map[string]interface{}, error)
	GraphDefinition() map[string]Graphs
}

// PluginWithPrefix is recommended interface
type PluginWithPrefix interface {
	Plugin
	MetricKeyPrefix() string
}

// MackerelPlugin is for mackerel-agent-plugin
type MackerelPlugin struct {
	Plugin
	Tempfile string
	diff     *bool
}

// NewMackerelPlugin returns new MackerelPlugin struct
func NewMackerelPlugin(plugin Plugin) MackerelPlugin {
	mp := MackerelPlugin{Plugin: plugin}
	return mp
}

func (h *MackerelPlugin) hasDiff() bool {
	if h.diff == nil {
		diff := false
		h.diff = &diff
	DiffCheck:
		for _, graph := range h.GraphDefinition() {
			for _, metric := range graph.Metrics {
				if metric.Diff {
					*h.diff = true
					break DiffCheck
				}
			}
		}
	}
	return *h.diff
}

func (h *MackerelPlugin) printValue(w io.Writer, key string, value interface{}, now time.Time) {
	switch value.(type) {
	case uint32:
		fmt.Fprintf(w, "%s\t%d\t%d\n", key, value.(uint32), now.Unix())
	case uint64:
		fmt.Fprintf(w, "%s\t%d\t%d\n", key, value.(uint64), now.Unix())
	case float64:
		if math.IsNaN(value.(float64)) || math.IsInf(value.(float64), 0) {
			log.Printf("Invalid value: key = %s, value = %f\n", key, value)
		} else {
			fmt.Fprintf(w, "%s\t%f\t%d\n", key, value.(float64), now.Unix())
		}
	}
}

func (h *MackerelPlugin) FetchLastValues() (map[string]interface{}, time.Time, error) {
	if !h.hasDiff() {
		return nil, time.Unix(0, 0), nil
	}
	lastTime := time.Now()

	f, err := os.Open(h.tempfilename())
	if err != nil {
		if os.IsNotExist(err) {
			return nil, lastTime, nil
		}
		return nil, lastTime, err
	}
	defer f.Close()

	stat := make(map[string]interface{})
	decoder := json.NewDecoder(f)
	err = decoder.Decode(&stat)
	switch stat["_lastTime"].(type) {
	case float64:
		lastTime = time.Unix(int64(stat["_lastTime"].(float64)), 0)
	case int64:
		lastTime = time.Unix(stat["_lastTime"].(int64), 0)
	}
	if err != nil {
		return stat, lastTime, err
	}
	return stat, lastTime, nil
}

func (h *MackerelPlugin) saveValues(values map[string]interface{}, now time.Time) error {
	if !h.hasDiff() {
		return nil
	}
	fname := h.tempfilename()
	f, err := os.Create(fname)
	if err != nil {
		return err
	}
	defer f.Close()

	values["_lastTime"] = now.Unix()
	encoder := json.NewEncoder(f)
	err = encoder.Encode(values)
	if err != nil {
		return err
	}

	return nil
}

func (h *MackerelPlugin) calcDiff(value float64, now time.Time, lastValue float64, lastTime time.Time) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0, errors.New("Too long duration")
	}

	diff := (value - lastValue) * 60 / float64(diffTime)

	if lastValue <= value {
		return diff, nil
	}
	return 0.0, errors.New("Counter seems to be reset.")
}

func (h *MackerelPlugin) calcDiffUint32(value uint32, now time.Time, lastValue uint32, lastTime time.Time, lastDiff float64) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0, errors.New("Too long duration")
	}

	diff := float64((value-lastValue)*60) / float64(diffTime)

	if lastValue <= value || diff < lastDiff*10 {
		return diff, nil
	}
	return 0.0, errors.New("Counter seems to be reset.")

}

func (h *MackerelPlugin) calcDiffUint64(value uint64, now time.Time, lastValue uint64, lastTime time.Time, lastDiff float64) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0, errors.New("Too long duration")
	}

	diff := float64((value-lastValue)*60) / float64(diffTime)

	if lastValue <= value || diff < lastDiff*10 {
		return diff, nil
	}
	return 0.0, errors.New("Counter seems to be reset.")
}

func (h *MackerelPlugin) tempfilename() string {
	if h.Tempfile == "" {
		h.Tempfile = h.generateTempfilePath(os.Args[0])
	}
	return h.Tempfile
}

var tempfileSanitizeReg = regexp.MustCompile(`[^A-Za-z0-9_.-]`)

// SetTempfileByBasename sets Tempfile under proper directory with specified basename.
func (h *MackerelPlugin) SetTempfileByBasename(base string) {
	h.Tempfile = filepath.Join(pluginutil.PluginWorkDir(), base)
}

func (h *MackerelPlugin) generateTempfilePath(path string) string {
	var prefix string
	if p, ok := h.Plugin.(PluginWithPrefix); ok {
		prefix = p.MetricKeyPrefix()
	} else {
		name := filepath.Base(path)
		prefix = strings.TrimPrefix(tempfileSanitizeReg.ReplaceAllString(name, "_"), "mackerel-plugin-")
	}
	filename := fmt.Sprintf("mackerel-plugin-%s", prefix)
	return filepath.Join(pluginutil.PluginWorkDir(), filename)
}

func (h *MackerelPlugin) formatValues(prefix string, metric Metrics, metricValues MetricValues, lastMetricValues MetricValues) {
	name := metric.Name
	if metric.AbsoluteName && len(prefix) > 0 {
		name = prefix + "." + name
	}
	value, ok := metricValues.Values[name]
	if !ok || value == nil {
		return
	}

	switch value.(type) {
	case string:
		switch metric.Type {
		case "uint32":
			value, _ = strconv.ParseUint(value.(string), 10, 32)
		case "uint64":
			value, _ = strconv.ParseUint(value.(string), 10, 64)
		default:
			value, _ = strconv.ParseFloat(value.(string), 64)
		}
	}

	if metric.Diff {
		_, ok := lastMetricValues.Values[name]
		if ok {
			var lastDiff float64
			if lastMetricValues.Values[".last_diff."+name] != nil {
				lastDiff = toFloat64(lastMetricValues.Values[".last_diff."+name])
			}
			var err error
			switch metric.Type {
			case "uint32":
				value, err = h.calcDiffUint32(toUint32(value), metricValues.Timestamp, toUint32(lastMetricValues.Values[name]), lastMetricValues.Timestamp, lastDiff)
			case "uint64":
				value, err = h.calcDiffUint64(toUint64(value), metricValues.Timestamp, toUint64(lastMetricValues.Values[name]), lastMetricValues.Timestamp, lastDiff)
			default:
				value, err = h.calcDiff(toFloat64(value), metricValues.Timestamp, toFloat64(lastMetricValues.Values[name]), lastMetricValues.Timestamp)
			}
			if err != nil {
				log.Println("OutputValues: ", err)
				return
			}
			metricValues.Values[".last_diff."+name] = value
		} else {
			log.Printf("%s does not exist at last fetch\n", name)
			return
		}
	}

	if metric.Scale != 0 {
		switch metric.Type {
		case "uint32":
			value = toUint32(value) * uint32(metric.Scale)
		case "uint64":
			value = toUint64(value) * uint64(metric.Scale)
		default:
			value = toFloat64(value) * metric.Scale
		}
	}

	metricNames := []string{}
	if p, ok := h.Plugin.(PluginWithPrefix); ok {
		metricNames = append(metricNames, p.MetricKeyPrefix())
	}
	if len(prefix) > 0 {
		metricNames = append(metricNames, prefix)
	}
	metricNames = append(metricNames, metric.Name)
	h.printValue(os.Stdout, strings.Join(metricNames, "."), value, metricValues.Timestamp)
}

func (h *MackerelPlugin) formatValuesWithWildcard(prefix string, metric Metrics, metricValues MetricValues, lastMetricValues MetricValues) {
	regexpStr := `\A` + prefix + "." + metric.Name
	regexpStr = strings.Replace(regexpStr, ".", "\\.", -1)
	regexpStr = strings.Replace(regexpStr, "*", "[-a-zA-Z0-9_]+", -1)
	regexpStr = strings.Replace(regexpStr, "#", "[-a-zA-Z0-9_]+", -1)
	re, err := regexp.Compile(regexpStr)
	if err != nil {
		log.Fatalln("Failed to compile regexp: ", err)
	}
	for k := range metricValues.Values {
		if re.MatchString(k) {
			metricEach := metric
			metricEach.Name = k
			h.formatValues("", metricEach, metricValues, lastMetricValues)
		}
	}
}

// Run the plugin
func (h *MackerelPlugin) Run() {
	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		h.OutputDefinitions()
	} else {
		h.OutputValues()
	}
}

// OutputValues output the metrics
func (h *MackerelPlugin) OutputValues() {
	now := time.Now()
	stat, err := h.FetchMetrics()
	if err != nil {
		log.Fatalln("OutputValues: ", err)
	}

	lastStat, lastTime, err := h.FetchLastValues()
	if err != nil {
		log.Println("FetchLastValues (ignore):", err)
	}

	for key, graph := range h.GraphDefinition() {
		for _, metric := range graph.Metrics {
			if strings.ContainsAny(key+metric.Name, "*#") {
				h.formatValuesWithWildcard(key, metric, MetricValues{Values: stat, Timestamp: now}, MetricValues{Values: lastStat, Timestamp: lastTime})
			} else {
				h.formatValues(key, metric, MetricValues{Values: stat, Timestamp: now}, MetricValues{Values: lastStat, Timestamp: lastTime})
			}
		}
	}

	err = h.saveValues(stat, now)
	if err != nil {
		log.Fatalln("saveValues: ", err)
	}
}

// GraphDef represents graph definitions
type GraphDef struct {
	Graphs map[string]Graphs `json:"graphs"`
}

func title(s string) string {
	r := strings.NewReplacer(".", " ", "_", " ")
	return strings.Title(r.Replace(s))
}

// OutputDefinitions outputs graph definitions
func (h *MackerelPlugin) OutputDefinitions() {
	fmt.Println("# mackerel-agent-plugin")
	graphs := make(map[string]Graphs)
	for key, graph := range h.GraphDefinition() {
		g := graph
		k := key
		if p, ok := h.Plugin.(PluginWithPrefix); ok {
			prefix := p.MetricKeyPrefix()
			if k == "" {
				k = prefix
			} else {
				k = prefix + "." + k
			}
		}
		if g.Label == "" {
			g.Label = title(k)
		}
		metrics := []Metrics{}
		for _, v := range g.Metrics {
			if v.Label == "" {
				v.Label = title(v.Name)
			}
			metrics = append(metrics, v)
		}
		g.Metrics = metrics
		graphs[k] = g
	}
	var graphdef GraphDef
	graphdef.Graphs = graphs
	b, err := json.Marshal(graphdef)
	if err != nil {
		log.Fatalln("OutputDefinitions: ", err)
	}
	fmt.Println(string(b))
}

func toUint32(value interface{}) uint32 {
	var ret uint32
	switch value.(type) {
	case uint32:
		ret = value.(uint32)
	case uint64:
		ret = uint32(value.(uint64))
	case float64:
		ret = uint32(value.(float64))
	case string:
		v, err := strconv.ParseUint(value.(string), 10, 32)
		if err == nil {
			ret = uint32(v)
		}
	}
	return ret
}

func toUint64(value interface{}) uint64 {
	var ret uint64
	switch value.(type) {
	case uint32:
		ret = uint64(value.(uint32))
	case uint64:
		ret = value.(uint64)
	case float64:
		ret = uint64(value.(float64))
	case string:
		ret, _ = strconv.ParseUint(value.(string), 10, 64)
	}
	return ret
}

func toFloat64(value interface{}) float64 {
	var ret float64
	switch value.(type) {
	case uint32:
		ret = float64(value.(uint32))
	case uint64:
		ret = float64(value.(uint64))
	case float64:
		ret = value.(float64)
	case string:
		ret, _ = strconv.ParseFloat(value.(string), 64)
	}
	return ret
}
<commit_msg>FetchLastValues now returns MetricValues<commit_after>package mackerelplugin

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"math"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/mackerelio/golib/pluginutil"
)

// Metrics represents definition of a metric
type Metrics struct {
	Name         string  `json:"name"`
	Label        string  `json:"label"`
	Diff         bool    `json:"-"`
	Type         string  `json:"-"`
	Stacked      bool    `json:"stacked"`
	Scale        float64 `json:"-"`
	AbsoluteName bool    `json:"-"`
}

// Graphs represents definition of a graph
type Graphs struct {
	Label   string    `json:"label"`
	Unit    string    `json:"unit"`
	Metrics []Metrics `json:"metrics"`
}

// MetricValues represents a collection of metric values and its timestamp
type MetricValues struct {
	Values    map[string]interface{}
	Timestamp time.Time
}

// Plugin is old interface of mackerel-plugin
type Plugin interface {
	FetchMetrics() (map[string]interface{}, error)
	GraphDefinition() map[string]Graphs
}

// PluginWithPrefix is recommended interface
type PluginWithPrefix interface {
	Plugin
	MetricKeyPrefix() string
}

// MackerelPlugin is for mackerel-agent-plugin
type MackerelPlugin struct {
	Plugin
	Tempfile string
	diff     *bool
}

// NewMackerelPlugin returns new MackerelPlugin struct
func NewMackerelPlugin(plugin Plugin) MackerelPlugin {
	mp := MackerelPlugin{Plugin: plugin}
	return mp
}

func (h *MackerelPlugin) hasDiff() bool {
	if h.diff == nil {
		diff := false
		h.diff = &diff
	DiffCheck:
		for _, graph := range h.GraphDefinition() {
			for _, metric := range graph.Metrics {
				if metric.Diff {
					*h.diff = true
					break DiffCheck
				}
			}
		}
	}
	return *h.diff
}

func (h *MackerelPlugin) printValue(w io.Writer, key string, value interface{}, now time.Time) {
	switch value.(type) {
	case uint32:
		fmt.Fprintf(w, "%s\t%d\t%d\n", key, value.(uint32), now.Unix())
	case uint64:
		fmt.Fprintf(w, "%s\t%d\t%d\n", key, value.(uint64), now.Unix())
	case float64:
		if math.IsNaN(value.(float64)) || math.IsInf(value.(float64), 0) {
			log.Printf("Invalid value: key = %s, value = %f\n", key, value)
		} else {
			fmt.Fprintf(w, "%s\t%f\t%d\n", key, value.(float64), now.Unix())
		}
	}
}

func (h *MackerelPlugin) FetchLastValues() (metricValues MetricValues, err error) {
	if !h.hasDiff() {
		return
	}
	metricValues.Timestamp = time.Now()

	f, err := os.Open(h.tempfilename())
	if err != nil {
		if os.IsNotExist(err) {
			return
		}
		return
	}
	defer f.Close()

	decoder := json.NewDecoder(f)
	err = decoder.Decode(&metricValues.Values)
	switch metricValues.Values["_lastTime"].(type) {
	case float64:
		metricValues.Timestamp = time.Unix(int64(metricValues.Values["_lastTime"].(float64)), 0)
	case int64:
		metricValues.Timestamp = time.Unix(metricValues.Values["_lastTime"].(int64), 0)
	}
	if err != nil {
		return
	}
	return
}

func (h *MackerelPlugin) saveValues(values map[string]interface{}, now time.Time) error {
	if !h.hasDiff() {
		return nil
	}
	fname := h.tempfilename()
	f, err := os.Create(fname)
	if err != nil {
		return err
	}
	defer f.Close()

	values["_lastTime"] = now.Unix()
	encoder := json.NewEncoder(f)
	err = encoder.Encode(values)
	if err != nil {
		return err
	}

	return nil
}

func (h *MackerelPlugin) calcDiff(value float64, now time.Time, lastValue float64, lastTime time.Time) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0, errors.New("Too long duration")
	}

	diff := (value - lastValue) * 60 / float64(diffTime)

	if lastValue <= value {
		return diff, nil
	}
	return 0.0, errors.New("Counter seems to be reset.")
}

func (h *MackerelPlugin) calcDiffUint32(value uint32, now time.Time, lastValue uint32, lastTime time.Time, lastDiff float64) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0, errors.New("Too long duration")
	}

	diff := float64((value-lastValue)*60) / float64(diffTime)

	if lastValue <= value || diff < lastDiff*10 {
		return diff, nil
	}
	return 0.0, errors.New("Counter seems to be reset.")

}

func (h *MackerelPlugin) calcDiffUint64(value uint64, now time.Time, lastValue uint64, lastTime time.Time, lastDiff float64) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0, errors.New("Too long duration")
	}

	diff := float64((value-lastValue)*60) / float64(diffTime)

	if lastValue <= value || diff < lastDiff*10 {
		return diff, nil
	}
	return 0.0, errors.New("Counter seems to be reset.")
}

func (h *MackerelPlugin) tempfilename() string {
	if h.Tempfile == "" {
		h.Tempfile = h.generateTempfilePath(os.Args[0])
	}
	return h.Tempfile
}

var tempfileSanitizeReg = regexp.MustCompile(`[^A-Za-z0-9_.-]`)

// SetTempfileByBasename sets Tempfile under proper directory with specified basename.
func (h *MackerelPlugin) SetTempfileByBasename(base string) {
	h.Tempfile = filepath.Join(pluginutil.PluginWorkDir(), base)
}

func (h *MackerelPlugin) generateTempfilePath(path string) string {
	var prefix string
	if p, ok := h.Plugin.(PluginWithPrefix); ok {
		prefix = p.MetricKeyPrefix()
	} else {
		name := filepath.Base(path)
		prefix = strings.TrimPrefix(tempfileSanitizeReg.ReplaceAllString(name, "_"), "mackerel-plugin-")
	}
	filename := fmt.Sprintf("mackerel-plugin-%s", prefix)
	return filepath.Join(pluginutil.PluginWorkDir(), filename)
}

func (h *MackerelPlugin) formatValues(prefix string, metric Metrics, metricValues MetricValues, lastMetricValues MetricValues) {
	name := metric.Name
	if metric.AbsoluteName && len(prefix) > 0 {
		name = prefix + "." + name
	}
	value, ok := metricValues.Values[name]
	if !ok || value == nil {
		return
	}

	switch value.(type) {
	case string:
		switch metric.Type {
		case "uint32":
			value, _ = strconv.ParseUint(value.(string), 10, 32)
		case "uint64":
			value, _ = strconv.ParseUint(value.(string), 10, 64)
		default:
			value, _ = strconv.ParseFloat(value.(string), 64)
		}
	}

	if metric.Diff {
		_, ok := lastMetricValues.Values[name]
		if ok {
			var lastDiff float64
			if lastMetricValues.Values[".last_diff."+name] != nil {
				lastDiff = toFloat64(lastMetricValues.Values[".last_diff."+name])
			}
			var err error
			switch metric.Type {
			case "uint32":
				value, err = h.calcDiffUint32(toUint32(value), metricValues.Timestamp, toUint32(lastMetricValues.Values[name]), lastMetricValues.Timestamp, lastDiff)
			case "uint64":
				value, err = h.calcDiffUint64(toUint64(value), metricValues.Timestamp, toUint64(lastMetricValues.Values[name]), lastMetricValues.Timestamp, lastDiff)
			default:
				value, err = h.calcDiff(toFloat64(value), metricValues.Timestamp, toFloat64(lastMetricValues.Values[name]), lastMetricValues.Timestamp)
			}
			if err != nil {
				log.Println("OutputValues: ", err)
				return
			}
			metricValues.Values[".last_diff."+name] = value
		} else {
			log.Printf("%s does not exist at last fetch\n", name)
			return
		}
	}

	if metric.Scale != 0 {
		switch metric.Type {
		case "uint32":
			value = toUint32(value) * uint32(metric.Scale)
		case "uint64":
			value = toUint64(value) * uint64(metric.Scale)
		default:
			value = toFloat64(value) * metric.Scale
		}
	}

	metricNames := []string{}
	if p, ok := h.Plugin.(PluginWithPrefix); ok {
		metricNames = append(metricNames, p.MetricKeyPrefix())
	}
	if len(prefix) > 0 {
		metricNames = append(metricNames, prefix)
	}
	metricNames = append(metricNames, metric.Name)
	h.printValue(os.Stdout, strings.Join(metricNames, "."), value, metricValues.Timestamp)
}

func (h *MackerelPlugin) formatValuesWithWildcard(prefix string, metric Metrics, metricValues MetricValues, lastMetricValues MetricValues) {
	regexpStr := `\A` + prefix + "." + metric.Name
	regexpStr = strings.Replace(regexpStr, ".", "\\.", -1)
	regexpStr = strings.Replace(regexpStr, "*", "[-a-zA-Z0-9_]+", -1)
	regexpStr = strings.Replace(regexpStr, "#", "[-a-zA-Z0-9_]+", -1)
	re, err := regexp.Compile(regexpStr)
	if err != nil {
		log.Fatalln("Failed to compile regexp: ", err)
	}
	for k := range metricValues.Values {
		if re.MatchString(k) {
			metricEach := metric
			metricEach.Name = k
			h.formatValues("", metricEach, metricValues, lastMetricValues)
		}
	}
}

// Run the plugin
func (h *MackerelPlugin) Run() {
	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		h.OutputDefinitions()
	} else {
		h.OutputValues()
	}
}

// OutputValues output the metrics
func (h *MackerelPlugin) OutputValues() {
	now := time.Now()
	stat, err := h.FetchMetrics()
	if err != nil {
		log.Fatalln("OutputValues: ", err)
	}

	lastMetricValues, err := h.FetchLastValues()
	if err != nil {
		log.Println("FetchLastValues (ignore):", err)
	}

	for key, graph := range h.GraphDefinition() {
		for _, metric := range graph.Metrics {
			if strings.ContainsAny(key+metric.Name, "*#") {
				h.formatValuesWithWildcard(key, metric, MetricValues{Values: stat, Timestamp: now}, lastMetricValues)
			} else {
				h.formatValues(key, metric, MetricValues{Values: stat, Timestamp: now}, lastMetricValues)
			}
		}
	}

	err = h.saveValues(stat, now)
	if err != nil {
		log.Fatalln("saveValues: ", err)
	}
}

// GraphDef represents graph definitions
type GraphDef struct {
	Graphs map[string]Graphs `json:"graphs"`
}

func title(s string) string {
	r := strings.NewReplacer(".", " ", "_", " ")
	return strings.Title(r.Replace(s))
}

// OutputDefinitions outputs graph definitions
func (h *MackerelPlugin) OutputDefinitions() {
	fmt.Println("# mackerel-agent-plugin")
	graphs := make(map[string]Graphs)
	for key, graph := range h.GraphDefinition() {
		g := graph
		k := key
		if p, ok := h.Plugin.(PluginWithPrefix); ok {
			prefix := p.MetricKeyPrefix()
			if k == "" {
				k = prefix
			} else {
				k = prefix + "." + k
			}
		}
		if g.Label == "" {
			g.Label = title(k)
		}
		metrics := []Metrics{}
		for _, v := range g.Metrics {
			if v.Label == "" {
				v.Label = title(v.Name)
			}
			metrics = append(metrics, v)
		}
		g.Metrics = metrics
		graphs[k] = g
	}
	var graphdef GraphDef
	graphdef.Graphs = graphs
	b, err := json.Marshal(graphdef)
	if err != nil {
		log.Fatalln("OutputDefinitions: ", err)
	}
	fmt.Println(string(b))
}

func toUint32(value interface{}) uint32 {
	var ret uint32
	switch value.(type) {
	case uint32:
		ret = value.(uint32)
	case uint64:
		ret = uint32(value.(uint64))
	case float64:
		ret = uint32(value.(float64))
	case string:
		v, err := strconv.ParseUint(value.(string), 10, 32)
		if err == nil {
			ret = uint32(v)
		}
	}
	return ret
}

func toUint64(value interface{}) uint64 {
	var ret uint64
	switch value.(type) {
	case uint32:
		ret = uint64(value.(uint32))
	case uint64:
		ret = value.(uint64)
	case float64:
		ret = uint64(value.(float64))
	case string:
		ret, _ = strconv.ParseUint(value.(string), 10, 64)
	}
	return ret
}

func toFloat64(value interface{}) float64 {
	var ret float64
	switch value.(type) {
	case uint32:
		ret = float64(value.(uint32))
	case uint64:
		ret = float64(value.(uint64))
	case float64:
		ret = value.(float64)
	case string:
		ret, _ = strconv.ParseFloat(value.(string), 64)
	}
	return ret
}
<|endoftext|>


<commit_before>package mackerelplugin

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"time"
)

type Metrics struct {
	Name    string `json:"name"`
	Label   string `json:"label"`
	Diff    bool   `json:"diff"`
	Stacked bool   `json:"stacked"`
}

type Graphs struct {
	Label   string    `json:"label"`
	Unit    string    `json:"unit"`
	Metrics []Metrics `json:"metrics"`
}

type Plugin interface {
	FetchMetrics() (map[string]float64, error)
	GraphDefinition() map[string]Graphs
}

type MackerelPlugin struct {
	Plugin
	Tempfile string
}

func NewMackerelPlugin(plugin Plugin) MackerelPlugin {
	mp := MackerelPlugin{plugin, "/tmp/mackerel-plugin-default"}
	return mp
}

func (h *MackerelPlugin) printValue(w io.Writer, key string, value float64, now time.Time) {
	if value == float64(int(value)) {
		fmt.Fprintf(w, "%s\t%d\t%d\n", key, int(value), now.Unix())
	} else {
		fmt.Fprintf(w, "%s\t%f\t%d\n", key, value, now.Unix())
	}
}

func (h *MackerelPlugin) fetchLastValues() (map[string]float64, time.Time, error) {
	lastTime := time.Now()

	f, err := os.Open(h.Tempfilename())
	if err != nil {
		if os.IsNotExist(err) {
			return nil, lastTime, nil
		}
		return nil, lastTime, err
	}
	defer f.Close()

	stat := make(map[string]float64)
	decoder := json.NewDecoder(f)
	err = decoder.Decode(&stat)
	lastTime = time.Unix(int64(stat["_lastTime"]), 0)
	if err != nil {
		return stat, lastTime, err
	}
	return stat, lastTime, nil
}

func (h *MackerelPlugin) saveValues(values map[string]float64, now time.Time) error {
	f, err := os.Create(h.Tempfilename())
	if err != nil {
		return err
	}
	defer f.Close()

	values["_lastTime"] = float64(now.Unix())
	encoder := json.NewEncoder(f)
	err = encoder.Encode(values)
	if err != nil {
		return err
	}

	return nil
}

func (h *MackerelPlugin) calcDiff(value float64, now time.Time, lastValue float64, lastTime time.Time) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0, errors.New("Too long duration")
	}

	diff := (value - lastValue) * 60 / float64(diffTime)
	return diff, nil
}

func (h *MackerelPlugin) Tempfilename() string {
	return h.Tempfile
}

func (h *MackerelPlugin) OutputValues() {
	now := time.Now()
	stat, err := h.FetchMetrics()
	if err != nil {
		log.Fatalln("OutputValues: ", err)
	}

	lastStat, lastTime, err := h.fetchLastValues()
	if err != nil {
		log.Println("fetchLastValues (ignore):", err)
	}

	err = h.saveValues(stat, now)
	if err != nil {
		log.Fatalf("saveValues: ", err)
	}

	for key, graph := range h.GraphDefinition() {
		for _, metric := range graph.Metrics {
			value := stat[metric.Name]

			if metric.Diff {
				_, ok := lastStat[metric.Name]
				if ok {
					value, err = h.calcDiff(value, now, lastStat[metric.Name], lastTime)
					if err != nil {
						log.Println("OutputValues: ", err)
					}
				} else {
					log.Printf("%s is not exist at last fetch\n", metric.Name)
				}
			}

			h.printValue(os.Stdout, key+"."+metric.Name, value, now)
		}
	}
}

type GraphDef struct {
	Graphs map[string]Graphs `json:"graphs"`
}

func (h *MackerelPlugin) OutputDefinitions() {
	fmt.Println("# mackerel-agent-plugin")
	var graphs GraphDef
	graphs.Graphs = h.GraphDefinition()

	b, err := json.Marshal(graphs)
	if err != nil {
		log.Fatalln("OutputDefinitions: ", err)
	}
	fmt.Println(string(b))
}
<commit_msg>Add Scale to Metrics struct<commit_after>package mackerelplugin

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"time"
)

type Metrics struct {
	Name    string  `json:"name"`
	Label   string  `json:"label"`
	Diff    bool    `json:"diff"`
	Stacked bool    `json:"stacked"`
	Scale   float64 `json:"scale"`
}

type Graphs struct {
	Label   string    `json:"label"`
	Unit    string    `json:"unit"`
	Metrics []Metrics `json:"metrics"`
}

type Plugin interface {
	FetchMetrics() (map[string]float64, error)
	GraphDefinition() map[string]Graphs
}

type MackerelPlugin struct {
	Plugin
	Tempfile string
}

func NewMackerelPlugin(plugin Plugin) MackerelPlugin {
	mp := MackerelPlugin{plugin, "/tmp/mackerel-plugin-default"}
	return mp
}

func (h *MackerelPlugin) printValue(w io.Writer, key string, value float64, now time.Time) {
	if value == float64(int(value)) {
		fmt.Fprintf(w, "%s\t%d\t%d\n", key, int(value), now.Unix())
	} else {
		fmt.Fprintf(w, "%s\t%f\t%d\n", key, value, now.Unix())
	}
}

func (h *MackerelPlugin) fetchLastValues() (map[string]float64, time.Time, error) {
	lastTime := time.Now()

	f, err := os.Open(h.Tempfilename())
	if err != nil {
		if os.IsNotExist(err) {
			return nil, lastTime, nil
		}
		return nil, lastTime, err
	}
	defer f.Close()

	stat := make(map[string]float64)
	decoder := json.NewDecoder(f)
	err = decoder.Decode(&stat)
	lastTime = time.Unix(int64(stat["_lastTime"]), 0)
	if err != nil {
		return stat, lastTime, err
	}
	return stat, lastTime, nil
}

func (h *MackerelPlugin) saveValues(values map[string]float64, now time.Time) error {
	f, err := os.Create(h.Tempfilename())
	if err != nil {
		return err
	}
	defer f.Close()

	values["_lastTime"] = float64(now.Unix())
	encoder := json.NewEncoder(f)
	err = encoder.Encode(values)
	if err != nil {
		return err
	}

	return nil
}

func (h *MackerelPlugin) calcDiff(value float64, now time.Time, lastValue float64, lastTime time.Time) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0, errors.New("Too long duration")
	}

	diff := (value - lastValue) * 60 / float64(diffTime)
	return diff, nil
}

func (h *MackerelPlugin) Tempfilename() string {
	return h.Tempfile
}

func (h *MackerelPlugin) OutputValues() {
	now := time.Now()
	stat, err := h.FetchMetrics()
	if err != nil {
		log.Fatalln("OutputValues: ", err)
	}

	lastStat, lastTime, err := h.fetchLastValues()
	if err != nil {
		log.Println("fetchLastValues (ignore):", err)
	}

	err = h.saveValues(stat, now)
	if err != nil {
		log.Fatalf("saveValues: ", err)
	}

	for key, graph := range h.GraphDefinition() {
		for _, metric := range graph.Metrics {
			value := stat[metric.Name]

			if metric.Diff {
				_, ok := lastStat[metric.Name]
				if ok {
					value, err = h.calcDiff(value, now, lastStat[metric.Name], lastTime)
					if err != nil {
						log.Println("OutputValues: ", err)
					}
				} else {
					log.Printf("%s is not exist at last fetch\n", metric.Name)
				}
			}

			value *= metric.Scale

			h.printValue(os.Stdout, key+"."+metric.Name, value, now)
		}
	}
}

type GraphDef struct {
	Graphs map[string]Graphs `json:"graphs"`
}

func (h *MackerelPlugin) OutputDefinitions() {
	fmt.Println("# mackerel-agent-plugin")
	var graphs GraphDef
	graphs.Graphs = h.GraphDefinition()

	b, err := json.Marshal(graphs)
	if err != nil {
		log.Fatalln("OutputDefinitions: ", err)
	}
	fmt.Println(string(b))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"github.com/SudoQ/smArt/data"
	"os"
	"flag"

	//"encoding/base64"
	"image"
	"log"
	//"strings"

	_ "image/gif"
	"image/png"
	_"image/jpeg"
	"time"
	"image/color"
	//"image/draw"
)

func loadImage(filename string) []*data.Data{
	// Decode the JPEG data. If reading from file, create a reader with

	reader, err := os.Open(filename)
	if err != nil {
	    log.Fatal(err)
	}
	defer reader.Close()

	//reader := base64.NewDecoder(base64.StdEncoding, strings.NewReader(data))
	m, _, err := image.Decode(reader)
	if err != nil {
		log.Fatal(err)
	}
	bounds := m.Bounds()

	dataSet := make([]*data.Data, 0)

	// Calculate a 16-bin histogram for m's red, green, blue and alpha components.
	//
	// An image's bounds do not necessarily start at (0, 0), so the two loops start
	// at bounds.Min.Y and bounds.Min.X. Looping over Y first and X second is more
	// likely to result in better memory access patterns than X first and Y second.
	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			r, g, b, _ := m.At(x, y).RGBA()
			// A color's RGBA method returns values in the range [0, 65535].
			// Shifting by 8 reduces this to the range [0, 255].
			a0 := float64(r>>8)
			a1 := float64(g>>8)
			a2 := float64(b>>8)
			d := data.New([]float64{a0, a1, a2}, 5)
			dataSet = append(dataSet, d)

		}
	}
	// Print the results.
	return dataSet
}

func saveCentroids(centroids []*data.Data, filename string){
	outfile, err := os.Create(filename)
	if err != nil {
		log.Println(err)
		return
	}
	imgRect := image.Rect(0, 0, 100, 100)
	img := image.NewRGBA(imgRect)
	//draw.Draw(img, img.Bounds(), &image.Uniform{color.White}, image.ZP, draw.Src)
	for y := 0; y < 100; y += 1 {
		ci := int((float64(y)/100)*5);
		r := uint8(centroids[ci].Attributes[0])
		g := uint8(centroids[ci].Attributes[1])
		b := uint8(centroids[ci].Attributes[2])
		for x := 0; x < 100; x += 1 {
			img.SetRGBA(x,y,color.RGBA{r,g,b,255})
			//fill := &image.Uniform{color.RGBA{r, g, b, 255}}
			//draw.Draw(img, image.Rect(x, y, x+10, y+10), fill, image.ZP, draw.Src)
		}
	}

	err = png.Encode(outfile, img)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	fmt.Printf("Generated image to %s\n",filename)
}

func applyModel(inputFilename, outputFilename string, centroids []*data.Data){
	reader, err := os.Open(inputFilename)
	if err != nil {
	    log.Fatal(err)
	}
	defer reader.Close()

	//reader := base64.NewDecoder(base64.StdEncoding, strings.NewReader(data))
	m, _, err := image.Decode(reader)
	if err != nil {
		log.Fatal(err)
	}
	bounds := m.Bounds()
	writer, err := os.Create(outputFilename)
	if err != nil {
		log.Println(err)
		return
	}
	defer writer.Close()

	imgRect := image.Rect(bounds.Min.X, bounds.Min.Y, bounds.Max.X, bounds.Max.Y)
	img := image.NewRGBA(imgRect)

	// An image's bounds do not necessarily start at (0, 0), so the two loops start
	// at bounds.Min.Y and bounds.Min.X. Looping over Y first and X second is more
	// likely to result in better memory access patterns than X first and Y second.
	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			r, g, b, _ := m.At(x, y).RGBA()
			// A color's RGBA method returns values in the range [0, 65535].
			// Shifting by 8 reduces this to the range [0, 255].
			a0 := float64(r>>8)
			a1 := float64(g>>8)
			a2 := float64(b>>8)
			d := data.New([]float64{a0, a1, a2}, 5)
			d.UpdateClassification(centroids)
			class := d.Classification
			b0 := uint8(centroids[class].Attributes[0])
			b1 := uint8(centroids[class].Attributes[1])
			b2 := uint8(centroids[class].Attributes[2])
			img.SetRGBA(x,y,color.RGBA{b0, b1, b2, 255})
		}
	}
	err = png.Encode(writer, img)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	fmt.Printf("Generated image to %s\n",outputFilename)
}

var inFilename string
var paletteFilename string
var outFilename string
func init() {
	flag.StringVar(&inFilename, "infile", "default_input.png", "Input filename")
	flag.StringVar(&paletteFilename, "palette", "default_palette.png", "Output palette filename")
	flag.StringVar(&outFilename, "outfile", "default_output.png", "Output filename")
}

func main(){
	flag.Parse()
	dataSet := loadImage(inFilename)
	fmt.Println(len(dataSet))
	rand.Seed(time.Now().UTC().UnixNano())

	// Read image

	K := 5
	centroids := []*data.Data{
		data.New([]float64{0.0, 0.0, 0.0}, K),  // Black
		data.New([]float64{255.0, 255.0, 255.0}, K),	// White
		data.New([]float64{255.0, 0.0, 0.0}, K),	// Red
		data.New([]float64{0.0, 255.0, 0.0}, K),	// Green
		data.New([]float64{0.0, 0.0, 255.0}, K),	// Blue
	}

	for n:=0; n<20; n++ {
		for _, v := range(dataSet) {
			v.UpdateClassification(centroids)
		}

		for _, v := range(centroids){
			v.UpdateClassification(centroids)
		}

		cCount := make(map[int]float64)
		for _, dataItem := range(dataSet) {
			ci := dataItem.Classification
			cCount[ci] += 1.0
			centroids[ci].Waverage(dataItem, 1.0/cCount[ci])
		}

	}
	for i, item := range(centroids) {
		r := uint32(item.Attributes[0])
		g := uint32(item.Attributes[1])
		b := uint32(item.Attributes[2])
		fmt.Printf("Color %d: (%d, %d, %d)\n", i,r,g,b)
	}
	saveCentroids(centroids, paletteFilename)
	applyModel(inFilename,outFilename,centroids)
}
<commit_msg>Updated input flags and cleanup<commit_after>package main

import (
	"fmt"
	"math/rand"
	"github.com/SudoQ/smArt/data"
	"os"
	"flag"

	//"encoding/base64"
	"image"
	"log"
	//"strings"

	_ "image/gif"
	"image/png"
	_"image/jpeg"
	"time"
	"image/color"
	//"image/draw"
)

func loadImage(filename string) []*data.Data{
	// Decode the JPEG data. If reading from file, create a reader with

	reader, err := os.Open(filename)
	if err != nil {
	    log.Fatal(err)
	}
	defer reader.Close()

	m, _, err := image.Decode(reader)
	if err != nil {
		log.Fatal(err)
	}
	bounds := m.Bounds()

	dataSet := make([]*data.Data, 0)

	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			r, g, b, _ := m.At(x, y).RGBA()
			// A color's RGBA method returns values in the range [0, 65535].
			// Shifting by 8 reduces this to the range [0, 255].
			a0 := float64(r>>8)
			a1 := float64(g>>8)
			a2 := float64(b>>8)
			d := data.New([]float64{a0, a1, a2}, 5)
			dataSet = append(dataSet, d)

		}
	}
	return dataSet
}

func saveCentroids(centroids []*data.Data, filename string){
	outfile, err := os.Create(filename)
	if err != nil {
		log.Println(err)
		return
	}
	imgRect := image.Rect(0, 0, 100, 100)
	img := image.NewRGBA(imgRect)
	for y := 0; y < 100; y += 1 {
		ci := int((float64(y)/100)*5);
		r := uint8(centroids[ci].Attributes[0])
		g := uint8(centroids[ci].Attributes[1])
		b := uint8(centroids[ci].Attributes[2])
		for x := 0; x < 100; x += 1 {
			img.SetRGBA(x,y,color.RGBA{r,g,b,255})
		}
	}

	err = png.Encode(outfile, img)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	fmt.Printf("Generated image to %s\n",filename)
}

func applyModel(inputFilename, outputFilename string, centroids []*data.Data){
	reader, err := os.Open(inputFilename)
	if err != nil {
	    log.Fatal(err)
	}
	defer reader.Close()

	m, _, err := image.Decode(reader)
	if err != nil {
		log.Fatal(err)
	}
	bounds := m.Bounds()
	writer, err := os.Create(outputFilename)
	if err != nil {
		log.Println(err)
		return
	}
	defer writer.Close()

	imgRect := image.Rect(bounds.Min.X, bounds.Min.Y, bounds.Max.X, bounds.Max.Y)
	img := image.NewRGBA(imgRect)

	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			r, g, b, _ := m.At(x, y).RGBA()
			a0 := float64(r>>8)
			a1 := float64(g>>8)
			a2 := float64(b>>8)
			d := data.New([]float64{a0, a1, a2}, 5)
			d.UpdateClassification(centroids)
			class := d.Classification
			b0 := uint8(centroids[class].Attributes[0])
			b1 := uint8(centroids[class].Attributes[1])
			b2 := uint8(centroids[class].Attributes[2])
			img.SetRGBA(x,y,color.RGBA{b0, b1, b2, 255})
		}
	}
	err = png.Encode(writer, img)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	fmt.Printf("Generated image to %s\n",outputFilename)
}

var inFilename string
var paletteFilename string
var outFilename string
func init() {
	flag.StringVar(&inFilename, "in", "default_input.png", "Input filename")
	flag.StringVar(&paletteFilename, "pal", "default_palette.png", "Output palette filename")
	flag.StringVar(&outFilename, "out", "default_output.png", "Output filename")
}

func main(){
	flag.Parse()
	dataSet := loadImage(inFilename)
	fmt.Println(len(dataSet))
	rand.Seed(time.Now().UTC().UnixNano())

	// Read image

	K := 5
	centroids := []*data.Data{
		data.New([]float64{0.0, 0.0, 0.0}, K),  // Black
		data.New([]float64{255.0, 255.0, 255.0}, K),	// White
		data.New([]float64{255.0, 0.0, 0.0}, K),	// Red
		data.New([]float64{0.0, 255.0, 0.0}, K),	// Green
		data.New([]float64{0.0, 0.0, 255.0}, K),	// Blue
	}

	for n:=0; n<20; n++ {
		for _, v := range(dataSet) {
			v.UpdateClassification(centroids)
		}

		for _, v := range(centroids){
			v.UpdateClassification(centroids)
		}

		cCount := make(map[int]float64)
		for _, dataItem := range(dataSet) {
			ci := dataItem.Classification
			cCount[ci] += 1.0
			centroids[ci].Waverage(dataItem, 1.0/cCount[ci])
		}

	}
	for i, item := range(centroids) {
		r := uint32(item.Attributes[0])
		g := uint32(item.Attributes[1])
		b := uint32(item.Attributes[2])
		fmt.Printf("Color %d: (%d, %d, %d)\n", i,r,g,b)
	}
	saveCentroids(centroids, paletteFilename)
	applyModel(inFilename,outFilename,centroids)
}
<|endoftext|>