<commit_before>package kitserver

import (
	"errors"
	"fmt"
	"net/http"

	"golang.org/x/net/context"

	kitlog "github.com/go-kit/kit/log"

	"github.com/kolide/kolide-ose/datastore"
	"github.com/kolide/kolide-ose/kolide"
)

func login(ds kolide.UserStore, logger kitlog.Logger) http.HandlerFunc {
	ctx := context.Background()
	logger = kitlog.NewContext(logger).With("method", "login")
	return func(w http.ResponseWriter, r *http.Request) {
		username := r.FormValue("username")
		password := r.FormValue("password")
		if username == "" || password == "" {
			http.Redirect(w, r, "/", http.StatusFound)
			return
		}

		// retrieve user or respond with error
		user, err := getUser(ctx, ds, w, username, password)
		if err != nil {
			logger.Log("err", err, "user", username)
			return
		}

		// create session here

		encodeResponse(ctx, w, getUserResponse{
			ID:                 user.ID,
			Username:           user.Username,
			Name:               user.Name,
			Admin:              user.Admin,
			Enabled:            user.Enabled,
			NeedsPasswordReset: user.NeedsPasswordReset,
		})
		logger.Log("msg", "authenticated", "user", username, "id", user.ID)

	}

}

// gets user from datastore or responds with an authError
func getUser(ctx context.Context, ds kolide.UserStore, w http.ResponseWriter, username, password string) (*kolide.User, error) {
	user, err := ds.User(username)
	switch err {
	case nil:
	case datastore.ErrNotFound:
		encodeResponse(ctx, w, getUserResponse{
			Err: authError{
				message: fmt.Sprintf("user %s not found", username),
			},
		})
		return nil, err
	default:
		encodeResponse(ctx, w, getUserResponse{
			Err: errors.New("unknown error, try again later"),
		})
		return nil, err
	}
	err = user.ValidatePassword(password)
	if err != nil {
		encodeResponse(ctx, w, getUserResponse{
			Err: authError{
				message: fmt.Sprintf("unauthorized: invalid password for user %s", username),
			},
		})
		return nil, err
	}
	return user, nil
}

const noAuthRedirect = "/"

func logout(ds kolide.UserStore, logger kitlog.Logger) http.HandlerFunc {
	logger = kitlog.NewContext(logger).With("method", "logout")
	return func(w http.ResponseWriter, r *http.Request) {
		// delete session first
		var username string
		var user kolide.User
		// TODO

		// redirect
		http.Redirect(w, r, noAuthRedirect, http.StatusFound)
		logger.Log("msg", "loggedout", "user", username, "id", user.ID)
	}
}

func authMiddleware(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

		// all good to pass
		next.ServeHTTP(w, r)
	})
}

type authError struct {
	message string
}

func (e authError) Error() string {
	if e.message == "" {
		return "unauthorized"
	}
	return fmt.Sprintf("unauthorized: %s", e.message)
}

// viewerContext is a struct which represents the ability for an execution
// context to participate in certain actions. Most often, a ViewerContext is
// associated with an application user, but a ViewerContext can represent a
// variety of other execution contexts as well (script, test, etc). The main
// purpose of a ViewerContext is to assist in the authorization of sensitive
// actions.
type viewerContext struct {
	user *kolide.User
}

// IsAdmin indicates whether or not the current user can perform administrative
// actions.
func (vc *viewerContext) IsAdmin() bool {
	if vc.user != nil {
		return vc.user.Admin && vc.user.Enabled
	}
	return false
}

// UserID is a helper that enables quick access to the user ID of the current
// user.
func (vc *viewerContext) UserID() uint {
	if vc.user != nil {
		return vc.user.ID
	}
	return 0
}

// IsLoggedIn determines whether or not the current VC is attached to a user
// account
func (vc *viewerContext) IsLoggedIn() bool {
	return vc.user != nil && vc.user.Enabled
}

// CanPerformActions returns a bool indicating the current user's ability to
// perform the most basic actions on the site
func (vc *viewerContext) CanPerformActions() bool {
	return vc.IsLoggedIn() && !vc.user.NeedsPasswordReset
}

// CanPerformReadActionsOnUser returns a bool indicating the current user's
// ability to perform read actions on the given user
func (vc *viewerContext) CanPerformReadActionOnUser(u *kolide.User) bool {
	return vc.CanPerformActions() || (vc.IsLoggedIn() && vc.IsUserID(u.ID))
}

// IsUserID returns true if the given user id the same as the user which is
// represented by this ViewerContext
func (vc *viewerContext) IsUserID(id uint) bool {
	if vc.UserID() == id {
		return true
	}
	return false
}

// newViewerContext generates a ViewerContext given a user struct
func newViewerContext(user *kolide.User) *viewerContext {
	return &viewerContext{
		user: user,
	}
}

// EmptyVC is a utility which generates an empty ViewerContext. This is often
// used to represent users which are not logged in.
func emptyVC() *viewerContext {
	return &viewerContext{}
}

func vcFromID(ds kolide.UserStore, id uint) (*viewerContext, error) {
	user, err := ds.UserByID(id)
	if err != nil {
		return nil, err
	}
	return &viewerContext{user: user}, nil
}
<commit_msg>add WriteActions context<commit_after>package kitserver

import (
	"errors"
	"fmt"
	"net/http"

	"golang.org/x/net/context"

	kitlog "github.com/go-kit/kit/log"

	"github.com/kolide/kolide-ose/datastore"
	"github.com/kolide/kolide-ose/kolide"
)

func login(ds kolide.UserStore, logger kitlog.Logger) http.HandlerFunc {
	ctx := context.Background()
	logger = kitlog.NewContext(logger).With("method", "login")
	return func(w http.ResponseWriter, r *http.Request) {
		username := r.FormValue("username")
		password := r.FormValue("password")
		if username == "" || password == "" {
			http.Redirect(w, r, "/", http.StatusFound)
			return
		}

		// retrieve user or respond with error
		user, err := getUser(ctx, ds, w, username, password)
		if err != nil {
			logger.Log("err", err, "user", username)
			return
		}
		// create session here

		encodeResponse(ctx, w, getUserResponse{
			ID:                 user.ID,
			Username:           user.Username,
			Name:               user.Name,
			Admin:              user.Admin,
			Enabled:            user.Enabled,
			NeedsPasswordReset: user.NeedsPasswordReset,
		})
		logger.Log("msg", "authenticated", "user", username, "id", user.ID)

	}

}

// gets user from datastore or responds with an authError
func getUser(ctx context.Context, ds kolide.UserStore, w http.ResponseWriter, username, password string) (*kolide.User, error) {
	user, err := ds.User(username)
	switch err {
	case nil:
	case datastore.ErrNotFound:
		encodeResponse(ctx, w, getUserResponse{
			Err: authError{
				message: fmt.Sprintf("user %s not found", username),
			},
		})
		return nil, err
	default:
		encodeResponse(ctx, w, getUserResponse{
			Err: errors.New("unknown error, try again later"),
		})
		return nil, err
	}
	err = user.ValidatePassword(password)
	if err != nil {
		encodeResponse(ctx, w, getUserResponse{
			Err: authError{
				message: fmt.Sprintf("unauthorized: invalid password for user %s", username),
			},
		})
		return nil, err
	}
	return user, nil
}

const noAuthRedirect = "/"

func logout(ds kolide.UserStore, logger kitlog.Logger) http.HandlerFunc {
	logger = kitlog.NewContext(logger).With("method", "logout")
	return func(w http.ResponseWriter, r *http.Request) {
		// delete session first
		var username string
		var user kolide.User
		// TODO

		// redirect
		http.Redirect(w, r, noAuthRedirect, http.StatusFound)
		logger.Log("msg", "loggedout", "user", username, "id", user.ID)
	}
}

func authMiddleware(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

		// all good to pass
		next.ServeHTTP(w, r)
	})
}

type authError struct {
	message string
}

func (e authError) Error() string {
	if e.message == "" {
		return "unauthorized"
	}
	return fmt.Sprintf("unauthorized: %s", e.message)
}

// viewerContext is a struct which represents the ability for an execution
// context to participate in certain actions. Most often, a ViewerContext is
// associated with an application user, but a ViewerContext can represent a
// variety of other execution contexts as well (script, test, etc). The main
// purpose of a ViewerContext is to assist in the authorization of sensitive
// actions.
type viewerContext struct {
	user *kolide.User
}

// IsAdmin indicates whether or not the current user can perform administrative
// actions.
func (vc *viewerContext) IsAdmin() bool {
	if vc.user != nil {
		return vc.user.Admin && vc.user.Enabled
	}
	return false
}

// UserID is a helper that enables quick access to the user ID of the current
// user.
func (vc *viewerContext) UserID() uint {
	if vc.user != nil {
		return vc.user.ID
	}
	return 0
}

// IsLoggedIn determines whether or not the current VC is attached to a user
// account
func (vc *viewerContext) IsLoggedIn() bool {
	return vc.user != nil && vc.user.Enabled
}

// CanPerformActions returns a bool indicating the current user's ability to
// perform the most basic actions on the site
func (vc *viewerContext) CanPerformActions() bool {
	return vc.IsLoggedIn() && !vc.user.NeedsPasswordReset
}

// CanPerformReadActionsOnUser returns a bool indicating the current user's
// ability to perform read actions on the given user
func (vc *viewerContext) CanPerformReadActionOnUser(u *kolide.User) bool {
	return vc.CanPerformActions() || (vc.IsLoggedIn() && vc.IsUserID(u.ID))
}

// CanPerformWriteActionOnUser returns a bool indicating the current user's
// ability to perform write actions on the given user
func (vc *viewerContext) CanPerformWriteActionOnUser(u *kolide.User) bool {
	return vc.CanPerformActions() && (vc.IsUserID(u.ID) || vc.IsAdmin())
}

// IsUserID returns true if the given user id the same as the user which is
// represented by this ViewerContext
func (vc *viewerContext) IsUserID(id uint) bool {
	if vc.UserID() == id {
		return true
	}
	return false
}

// newViewerContext generates a ViewerContext given a user struct
func newViewerContext(user *kolide.User) *viewerContext {
	return &viewerContext{
		user: user,
	}
}

// EmptyVC is a utility which generates an empty ViewerContext. This is often
// used to represent users which are not logged in.
func emptyVC() *viewerContext {
	return &viewerContext{}
}

func vcFromID(ds kolide.UserStore, id uint) (*viewerContext, error) {
	user, err := ds.UserByID(id)
	if err != nil {
		return nil, err
	}
	return &viewerContext{user: user}, nil
}
<|endoftext|>


<commit_before>package gateway

import (
	"fmt"
	"net/http"
	"time"

	"github.com/TykTechnologies/leakybucket"
	"github.com/TykTechnologies/leakybucket/memorycache"
	"github.com/TykTechnologies/tyk/config"
	"github.com/TykTechnologies/tyk/storage"
	"github.com/TykTechnologies/tyk/user"
)

type PublicSession struct {
	Quota struct {
		QuotaMax       int64 `json:"quota_max"`
		QuotaRemaining int64 `json:"quota_remaining"`
		QuotaRenews    int64 `json:"quota_renews"`
	} `json:"quota"`
	RateLimit struct {
		Rate float64 `json:"requests"`
		Per  float64 `json:"per_unit"`
	} `json:"rate_limit"`
}

const (
	QuotaKeyPrefix     = "quota-"
	RateLimitKeyPrefix = "rate-limit-"
)

// SessionLimiter is the rate limiter for the API, use ForwardMessage() to
// check if a message should pass through or not
type SessionLimiter struct {
	bucketStore leakybucket.Storage
}

func (l *SessionLimiter) doRollingWindowWrite(key, rateLimiterKey, rateLimiterSentinelKey string,
	currentSession *user.SessionState,
	store storage.Handler,
	globalConf *config.Config,
	apiLimit *user.APILimit, dryRun bool) bool {

	var per, rate float64

	if apiLimit != nil { // respect limit on API level
		per = apiLimit.Per
		rate = apiLimit.Rate
	} else {
		per = currentSession.Per
		rate = currentSession.Rate
	}

	log.Debug("[RATELIMIT] Inbound raw key is: ", key)
	log.Debug("[RATELIMIT] Rate limiter key is: ", rateLimiterKey)
	pipeline := globalConf.EnableNonTransactionalRateLimiter

	var ratePerPeriodNow int
	if dryRun {
		ratePerPeriodNow, _ = store.GetRollingWindow(rateLimiterKey, int64(per), pipeline)
	} else {
		ratePerPeriodNow, _ = store.SetRollingWindow(rateLimiterKey, int64(per), "-1", pipeline)
	}

	//log.Info("Num Requests: ", ratePerPeriodNow)

	// Subtract by 1 because of the delayed add in the window
	subtractor := 1
	if globalConf.EnableSentinelRateLimiter {
		// and another subtraction because of the preemptive limit
		subtractor = 2
	}
	// The test TestRateLimitForAPIAndRateLimitAndQuotaCheck
	// will only work with ththese two lines here
	//log.Info("break: ", (int(currentSession.Rate) - subtractor))
	if ratePerPeriodNow > int(rate)-subtractor {
		// Set a sentinel value with expire
		if globalConf.EnableSentinelRateLimiter {
			if !dryRun {
				store.SetRawKey(rateLimiterSentinelKey, "1", int64(per))
			}
		}
		return true
	}

	return false
}

type sessionFailReason uint

const (
	sessionFailNone sessionFailReason = iota
	sessionFailRateLimit
	sessionFailQuota
)

func (l *SessionLimiter) limitSentinel(
	currentSession *user.SessionState,
	key string,
	rateScope string,
	store storage.Handler,
	globalConf *config.Config,
	apiLimit *user.APILimit,
	dryRun bool,
) bool {
	rateLimiterKey := RateLimitKeyPrefix + rateScope + currentSession.KeyHash()
	rateLimiterSentinelKey := RateLimitKeyPrefix + rateScope + currentSession.KeyHash() + ".BLOCKED"

	go l.doRollingWindowWrite(key, rateLimiterKey, rateLimiterSentinelKey, currentSession, store, globalConf, apiLimit, dryRun)

	// Check sentinel
	_, sentinelActive := store.GetRawKey(rateLimiterSentinelKey)
	if sentinelActive == nil {
		// Sentinel is set, fail
		return true
	}
	return false
}
func (l *SessionLimiter) limitRedis(
	currentSession *user.SessionState,
	key string,
	rateScope string,
	store storage.Handler,
	globalConf *config.Config,
	apiLimit *user.APILimit,
	dryRun bool,
) bool {
	rateLimiterKey := RateLimitKeyPrefix + rateScope + currentSession.KeyHash()
	rateLimiterSentinelKey := RateLimitKeyPrefix + rateScope + currentSession.KeyHash() + ".BLOCKED"

	if l.doRollingWindowWrite(key, rateLimiterKey, rateLimiterSentinelKey, currentSession, store, globalConf, apiLimit, dryRun) {
		return true
	}
	return false
}
func (l *SessionLimiter) limitDRL(
	currentSession *user.SessionState,
	key string,
	rateScope string,
	apiLimit *user.APILimit,
	dryRun bool,
) bool {
	// In-memory limiter
	if l.bucketStore == nil {
		l.bucketStore = memorycache.New()
	}

	bucketKey := key + ":" + rateScope + currentSession.LastUpdated
	currRate := apiLimit.Rate
	per := apiLimit.Per

	// DRL will always overflow with more servers on low rates
	rate := uint(currRate * float64(DRLManager.RequestTokenValue))
	if rate < uint(DRLManager.CurrentTokenValue()) {
		rate = uint(DRLManager.CurrentTokenValue())
	}
	userBucket, err := l.bucketStore.Create(bucketKey, rate, time.Duration(per)*time.Second)
	if err != nil {
		log.Error("Failed to create bucket!")
		return true
	}

	if dryRun {
		// if userBucket is empty and not expired.
		if userBucket.Remaining() == 0 && time.Now().Before(userBucket.Reset()) {
			return true
		}
	} else {
		_, errF := userBucket.Add(uint(DRLManager.CurrentTokenValue()))
		if errF != nil {
			return true
		}
	}
	return false
}

func (sfr sessionFailReason) String() string {
	switch sfr {
	case sessionFailNone:
		return "sessionFailNone"
	case sessionFailRateLimit:
		return "sessionFailRateLimit"
	case sessionFailQuota:
		return "sessionFailQuota"
	default:
		return fmt.Sprintf("%d", uint(sfr))
	}
}

// ForwardMessage will enforce rate limiting, returning a non-zero
// sessionFailReason if session limits have been exceeded.
// Key values to manage rate are Rate and Per, e.g. Rate of 10 messages
// Per 10 seconds
func (l *SessionLimiter) ForwardMessage(r *http.Request, currentSession *user.SessionState, key string, store storage.Handler, enableRL, enableQ bool, globalConf *config.Config, apiID string, dryRun bool) sessionFailReason {
	// check for limit on API level (set to session by ApplyPolicies)
	var apiLimit *user.APILimit
	var allowanceScope string
	if len(currentSession.AccessRights) > 0 {
		if rights, ok := currentSession.AccessRights[apiID]; !ok {
			log.WithField("apiID", apiID).Debug("[RATE] unexpected apiID")
			return sessionFailRateLimit
		} else {
			apiLimit = rights.Limit
			allowanceScope = rights.AllowanceScope
		}
	}

	if apiLimit == nil {
		apiLimit = &user.APILimit{
			QuotaMax:           currentSession.QuotaMax,
			QuotaRenewalRate:   currentSession.QuotaRenewalRate,
			QuotaRenews:        currentSession.QuotaRenews,
			Rate:               currentSession.Rate,
			Per:                currentSession.Per,
			ThrottleInterval:   currentSession.ThrottleInterval,
			ThrottleRetryLimit: currentSession.ThrottleRetryLimit,
		}
	}

	if enableRL {
		rateScope := ""
		if allowanceScope != "" {
			rateScope = allowanceScope + "-"
		}
		if globalConf.EnableSentinelRateLimiter {
			if l.limitSentinel(currentSession, key, rateScope, store, globalConf, apiLimit, dryRun) {
				return sessionFailRateLimit
			}
		} else if globalConf.EnableRedisRollingLimiter {
			if l.limitRedis(currentSession, key, rateScope, store, globalConf, apiLimit, dryRun) {
				return sessionFailRateLimit
			}
		} else {
			var n float64
			if DRLManager.Servers != nil {
				n = float64(DRLManager.Servers.Count())
			}
			rate := apiLimit.Rate / apiLimit.Per
			c := globalConf.DRLThreshold
			if c == 0 {
				// defaults to 5
				c = 5
			}

			if n <= 1 || n*c > rate {
				// If we have 1 server, there is no need to strain redis at all the leaky
				// bucket algorithm will suffice.
				if l.limitDRL(currentSession, key, rateScope, apiLimit, dryRun) {
					return sessionFailRateLimit
				}
			} else {
				if l.limitRedis(currentSession, key, rateScope, store, globalConf, apiLimit, dryRun) {
					return sessionFailRateLimit
				}
			}
		}
	}

	if enableQ {
		if globalConf.LegacyEnableAllowanceCountdown {
			currentSession.Allowance--
		}

		if l.RedisQuotaExceeded(r, currentSession, allowanceScope, apiLimit, store) {
			return sessionFailQuota
		}
	}

	return sessionFailNone

}

func (l *SessionLimiter) RedisQuotaExceeded(r *http.Request, currentSession *user.SessionState, scope string, limit *user.APILimit, store storage.Handler) bool {
	// Unlimited?
	if limit.QuotaMax == -1 || limit.QuotaMax == 0 {
		// No quota set
		return false
	}

	quotaScope := ""
	if scope != "" {
		quotaScope = scope + "-"
	}

	rawKey := QuotaKeyPrefix + quotaScope + currentSession.KeyHash()
	quotaRenewalRate := limit.QuotaRenewalRate
	quotaRenews := limit.QuotaRenews
	quotaMax := limit.QuotaMax

	log.Debug("[QUOTA] Quota limiter key is: ", rawKey)
	log.Debug("Renewing with TTL: ", quotaRenewalRate)
	// INCR the key (If it equals 1 - set EXPIRE)
	qInt := store.IncrememntWithExpire(rawKey, quotaRenewalRate)

	// if the returned val is >= quota: block
	if qInt-1 >= quotaMax {
		renewalDate := time.Unix(quotaRenews, 0)
		log.Debug("Renewal Date is: ", renewalDate)
		log.Debug("As epoch: ", quotaRenews)
		log.Debug("Session: ", currentSession)
		log.Debug("Now:", time.Now())
		if time.Now().After(renewalDate) {
			// The renewal date is in the past, we should update the quota!
			// Also, this fixes legacy issues where there is no TTL on quota buckets
			log.Debug("Incorrect key expiry setting detected, correcting")
			go store.DeleteRawKey(rawKey)
			qInt = 1
		} else {
			// Renewal date is in the future and the quota is exceeded
			return true
		}

	}

	// If this is a new Quota period, ensure we let the end user know
	if qInt == 1 {
		quotaRenews = time.Now().Unix() + quotaRenewalRate
		ctxScheduleSessionUpdate(r)
	}

	// If not, pass and set the values of the session to quotamax - counter
	remaining := quotaMax - qInt
	if remaining < 0 {
		remaining = 0
	}

	for k, v := range currentSession.AccessRights {
		if v.Limit == nil {
			continue
		}

		if v.AllowanceScope == scope {
			v.Limit.QuotaRemaining = remaining
			v.Limit.QuotaRenews = quotaRenews
		}
		currentSession.AccessRights[k] = v
	}

	if scope == "" {
		currentSession.QuotaRemaining = remaining
		currentSession.QuotaRenews = quotaRenews
	}

	return false
}
<commit_msg>fix drl logic (#2938)<commit_after>package gateway

import (
	"fmt"
	"net/http"
	"time"

	"github.com/TykTechnologies/leakybucket"
	"github.com/TykTechnologies/leakybucket/memorycache"
	"github.com/TykTechnologies/tyk/config"
	"github.com/TykTechnologies/tyk/storage"
	"github.com/TykTechnologies/tyk/user"
)

type PublicSession struct {
	Quota struct {
		QuotaMax       int64 `json:"quota_max"`
		QuotaRemaining int64 `json:"quota_remaining"`
		QuotaRenews    int64 `json:"quota_renews"`
	} `json:"quota"`
	RateLimit struct {
		Rate float64 `json:"requests"`
		Per  float64 `json:"per_unit"`
	} `json:"rate_limit"`
}

const (
	QuotaKeyPrefix     = "quota-"
	RateLimitKeyPrefix = "rate-limit-"
)

// SessionLimiter is the rate limiter for the API, use ForwardMessage() to
// check if a message should pass through or not
type SessionLimiter struct {
	bucketStore leakybucket.Storage
}

func (l *SessionLimiter) doRollingWindowWrite(key, rateLimiterKey, rateLimiterSentinelKey string,
	currentSession *user.SessionState,
	store storage.Handler,
	globalConf *config.Config,
	apiLimit *user.APILimit, dryRun bool) bool {

	var per, rate float64

	if apiLimit != nil { // respect limit on API level
		per = apiLimit.Per
		rate = apiLimit.Rate
	} else {
		per = currentSession.Per
		rate = currentSession.Rate
	}

	log.Debug("[RATELIMIT] Inbound raw key is: ", key)
	log.Debug("[RATELIMIT] Rate limiter key is: ", rateLimiterKey)
	pipeline := globalConf.EnableNonTransactionalRateLimiter

	var ratePerPeriodNow int
	if dryRun {
		ratePerPeriodNow, _ = store.GetRollingWindow(rateLimiterKey, int64(per), pipeline)
	} else {
		ratePerPeriodNow, _ = store.SetRollingWindow(rateLimiterKey, int64(per), "-1", pipeline)
	}

	//log.Info("Num Requests: ", ratePerPeriodNow)

	// Subtract by 1 because of the delayed add in the window
	subtractor := 1
	if globalConf.EnableSentinelRateLimiter {
		// and another subtraction because of the preemptive limit
		subtractor = 2
	}
	// The test TestRateLimitForAPIAndRateLimitAndQuotaCheck
	// will only work with ththese two lines here
	//log.Info("break: ", (int(currentSession.Rate) - subtractor))
	if ratePerPeriodNow > int(rate)-subtractor {
		// Set a sentinel value with expire
		if globalConf.EnableSentinelRateLimiter {
			if !dryRun {
				store.SetRawKey(rateLimiterSentinelKey, "1", int64(per))
			}
		}
		return true
	}

	return false
}

type sessionFailReason uint

const (
	sessionFailNone sessionFailReason = iota
	sessionFailRateLimit
	sessionFailQuota
)

func (l *SessionLimiter) limitSentinel(
	currentSession *user.SessionState,
	key string,
	rateScope string,
	store storage.Handler,
	globalConf *config.Config,
	apiLimit *user.APILimit,
	dryRun bool,
) bool {
	rateLimiterKey := RateLimitKeyPrefix + rateScope + currentSession.KeyHash()
	rateLimiterSentinelKey := RateLimitKeyPrefix + rateScope + currentSession.KeyHash() + ".BLOCKED"

	go l.doRollingWindowWrite(key, rateLimiterKey, rateLimiterSentinelKey, currentSession, store, globalConf, apiLimit, dryRun)

	// Check sentinel
	_, sentinelActive := store.GetRawKey(rateLimiterSentinelKey)
	if sentinelActive == nil {
		// Sentinel is set, fail
		return true
	}
	return false
}
func (l *SessionLimiter) limitRedis(
	currentSession *user.SessionState,
	key string,
	rateScope string,
	store storage.Handler,
	globalConf *config.Config,
	apiLimit *user.APILimit,
	dryRun bool,
) bool {
	rateLimiterKey := RateLimitKeyPrefix + rateScope + currentSession.KeyHash()
	rateLimiterSentinelKey := RateLimitKeyPrefix + rateScope + currentSession.KeyHash() + ".BLOCKED"

	if l.doRollingWindowWrite(key, rateLimiterKey, rateLimiterSentinelKey, currentSession, store, globalConf, apiLimit, dryRun) {
		return true
	}
	return false
}
func (l *SessionLimiter) limitDRL(
	currentSession *user.SessionState,
	key string,
	rateScope string,
	apiLimit *user.APILimit,
	dryRun bool,
) bool {
	// In-memory limiter
	if l.bucketStore == nil {
		l.bucketStore = memorycache.New()
	}

	bucketKey := key + ":" + rateScope + currentSession.LastUpdated
	currRate := apiLimit.Rate
	per := apiLimit.Per

	// DRL will always overflow with more servers on low rates
	rate := uint(currRate * float64(DRLManager.RequestTokenValue))
	if rate < uint(DRLManager.CurrentTokenValue()) {
		rate = uint(DRLManager.CurrentTokenValue())
	}
	userBucket, err := l.bucketStore.Create(bucketKey, rate, time.Duration(per)*time.Second)
	if err != nil {
		log.Error("Failed to create bucket!")
		return true
	}

	if dryRun {
		// if userBucket is empty and not expired.
		if userBucket.Remaining() == 0 && time.Now().Before(userBucket.Reset()) {
			return true
		}
	} else {
		_, errF := userBucket.Add(uint(DRLManager.CurrentTokenValue()))
		if errF != nil {
			return true
		}
	}
	return false
}

func (sfr sessionFailReason) String() string {
	switch sfr {
	case sessionFailNone:
		return "sessionFailNone"
	case sessionFailRateLimit:
		return "sessionFailRateLimit"
	case sessionFailQuota:
		return "sessionFailQuota"
	default:
		return fmt.Sprintf("%d", uint(sfr))
	}
}

// ForwardMessage will enforce rate limiting, returning a non-zero
// sessionFailReason if session limits have been exceeded.
// Key values to manage rate are Rate and Per, e.g. Rate of 10 messages
// Per 10 seconds
func (l *SessionLimiter) ForwardMessage(r *http.Request, currentSession *user.SessionState, key string, store storage.Handler, enableRL, enableQ bool, globalConf *config.Config, apiID string, dryRun bool) sessionFailReason {
	// check for limit on API level (set to session by ApplyPolicies)
	var apiLimit *user.APILimit
	var allowanceScope string
	if len(currentSession.AccessRights) > 0 {
		if rights, ok := currentSession.AccessRights[apiID]; !ok {
			log.WithField("apiID", apiID).Debug("[RATE] unexpected apiID")
			return sessionFailRateLimit
		} else {
			apiLimit = rights.Limit
			allowanceScope = rights.AllowanceScope
		}
	}

	if apiLimit == nil {
		apiLimit = &user.APILimit{
			QuotaMax:           currentSession.QuotaMax,
			QuotaRenewalRate:   currentSession.QuotaRenewalRate,
			QuotaRenews:        currentSession.QuotaRenews,
			Rate:               currentSession.Rate,
			Per:                currentSession.Per,
			ThrottleInterval:   currentSession.ThrottleInterval,
			ThrottleRetryLimit: currentSession.ThrottleRetryLimit,
		}
	}

	if enableRL {
		rateScope := ""
		if allowanceScope != "" {
			rateScope = allowanceScope + "-"
		}
		if globalConf.EnableSentinelRateLimiter {
			if l.limitSentinel(currentSession, key, rateScope, store, globalConf, apiLimit, dryRun) {
				return sessionFailRateLimit
			}
		} else if globalConf.EnableRedisRollingLimiter {
			if l.limitRedis(currentSession, key, rateScope, store, globalConf, apiLimit, dryRun) {
				return sessionFailRateLimit
			}
		} else {
			var n float64
			if DRLManager.Servers != nil {
				n = float64(DRLManager.Servers.Count())
			}
			rate := apiLimit.Rate / apiLimit.Per
			c := globalConf.DRLThreshold
			if c == 0 {
				// defaults to 5
				c = 5
			}

			if n <= 1 || n*c < rate {
				// If we have 1 server, there is no need to strain redis at all the leaky
				// bucket algorithm will suffice.
				if l.limitDRL(currentSession, key, rateScope, apiLimit, dryRun) {
					return sessionFailRateLimit
				}
			} else {
				if l.limitRedis(currentSession, key, rateScope, store, globalConf, apiLimit, dryRun) {
					return sessionFailRateLimit
				}
			}
		}
	}

	if enableQ {
		if globalConf.LegacyEnableAllowanceCountdown {
			currentSession.Allowance--
		}

		if l.RedisQuotaExceeded(r, currentSession, allowanceScope, apiLimit, store) {
			return sessionFailQuota
		}
	}

	return sessionFailNone

}

func (l *SessionLimiter) RedisQuotaExceeded(r *http.Request, currentSession *user.SessionState, scope string, limit *user.APILimit, store storage.Handler) bool {
	// Unlimited?
	if limit.QuotaMax == -1 || limit.QuotaMax == 0 {
		// No quota set
		return false
	}

	quotaScope := ""
	if scope != "" {
		quotaScope = scope + "-"
	}

	rawKey := QuotaKeyPrefix + quotaScope + currentSession.KeyHash()
	quotaRenewalRate := limit.QuotaRenewalRate
	quotaRenews := limit.QuotaRenews
	quotaMax := limit.QuotaMax

	log.Debug("[QUOTA] Quota limiter key is: ", rawKey)
	log.Debug("Renewing with TTL: ", quotaRenewalRate)
	// INCR the key (If it equals 1 - set EXPIRE)
	qInt := store.IncrememntWithExpire(rawKey, quotaRenewalRate)

	// if the returned val is >= quota: block
	if qInt-1 >= quotaMax {
		renewalDate := time.Unix(quotaRenews, 0)
		log.Debug("Renewal Date is: ", renewalDate)
		log.Debug("As epoch: ", quotaRenews)
		log.Debug("Session: ", currentSession)
		log.Debug("Now:", time.Now())
		if time.Now().After(renewalDate) {
			// The renewal date is in the past, we should update the quota!
			// Also, this fixes legacy issues where there is no TTL on quota buckets
			log.Debug("Incorrect key expiry setting detected, correcting")
			go store.DeleteRawKey(rawKey)
			qInt = 1
		} else {
			// Renewal date is in the future and the quota is exceeded
			return true
		}

	}

	// If this is a new Quota period, ensure we let the end user know
	if qInt == 1 {
		quotaRenews = time.Now().Unix() + quotaRenewalRate
		ctxScheduleSessionUpdate(r)
	}

	// If not, pass and set the values of the session to quotamax - counter
	remaining := quotaMax - qInt
	if remaining < 0 {
		remaining = 0
	}

	for k, v := range currentSession.AccessRights {
		if v.Limit == nil {
			continue
		}

		if v.AllowanceScope == scope {
			v.Limit.QuotaRemaining = remaining
			v.Limit.QuotaRenews = quotaRenews
		}
		currentSession.AccessRights[k] = v
	}

	if scope == "" {
		currentSession.QuotaRemaining = remaining
		currentSession.QuotaRenews = quotaRenews
	}

	return false
}
<|endoftext|>


<commit_before>package main

import "encoding/json"
import "github.com/jmhodges/levigo"
import "github.com/gokyle/kludge/common"
import "log"

var reqQ chan *common.Request

func startPool() {
	reqQ = make(chan *common.Request, reqBuf)
	for i := 0; i < poolSize; i++ {
		go requestHandler(i)
	}
}

func requestHandler(id int) {
	for {
		req, ok := <-reqQ
		if !ok {
			log.Printf("worker %d returns", id)
			return
		}
		req.Op.WID = id

		log.Printf("worker %d handling %s request", id,
			req.OpName())
		switch req.Op.OpCode {
		case common.OpGet:
			req.Resp <- store_get(req.Op)
		case common.OpSet:
			req.Resp <- store_set(req.Op)
		case common.OpDel:
			req.Resp <- store_del(req.Op)
		default:
			log.Printf("worker %d received invalid operation %d",
				id, req.Op.OpCode)
		}
	}
}

func store_get(op *common.Operation) (resp *common.Response) {
	ropts := levigo.NewReadOptions()
	ropts.SetVerifyChecksums(true)

	data, err := ldb.Get(ropts, op.Key)
	if err != nil {
		log.Printf("error handling get from worker %d: %s",
			op.WID, err.Error())
	} else {
		log.Printf("worker %d successfully completes GET", op.WID)
		resp = new(common.Response)
		resp.Body = data
	}
	return
}

func store_set(op *common.Operation) (resp *common.Response) {
	ropts := levigo.NewReadOptions()
	ropts.SetVerifyChecksums(true)

	data, err := ldb.Get(ropts, op.Key)
	if err != nil {
		log.Printf("worker %d failed to read key: %s", op.WID,
			err.Error())
		return
	}

	wopts := levigo.NewWriteOptions()
	wopts.SetSync(true)
	err = ldb.Put(wopts, op.Key, op.Val)
	if err != nil {
		log.Printf("worker %d failed to set key: %s", op.WID,
			err.Error())
		return
	} else {
		log.Printf("worker %d successfully wrote key", op.WID)
		resp = new(common.Response)
		resp.Body = data
	}
	return
}

func store_del(op *common.Operation) (resp *common.Response) {
	ropts := levigo.NewReadOptions()
	ropts.SetVerifyChecksums(true)

	data, err := ldb.Get(ropts, op.Key)
	if err != nil {
		log.Printf("worker %d failed to read key: %s", op.WID,
			err.Error())
		return
	}

	wopts := levigo.NewWriteOptions()
	wopts.SetSync(true)
	err = ldb.Delete(wopts, op.Key)
	if err != nil {
		log.Printf("worker %d failed to delete key: %s", op.WID,
			err.Error())
		return
	} else {
		if len(data) > 0 {
			resp.Body = []byte("ok")
		}
	}
	return
}

func store_lst(op *common.Operation) (resp *common.Response) {
        keys := make([]string, 0)
        ro := levigo.NewReadOptions()
        ro.SetFillCache(false)
        it := ldb.NewIterator(ro)
        for it = it; it.Valid(); it.Next() {
                keys = append(keys, string(it.Key()))
        }

        if err := it.GetError(); err != nil {
                log.Printf("worker %d failed to iterate over keys: %s",
                        op.WID, err.Error())
        } else {
                resp = new(common.Response)
                resp.Body, err = json.Marshal(keys)
                if err != nil {
                        log.Printf("worker %d failed to create JSON response: %s",
                                op.WID, err.Error())
                        resp.Body = []byte{}
                }
        }
        return
}
<commit_msg>go fmt<commit_after>package main

import "encoding/json"
import "github.com/jmhodges/levigo"
import "github.com/gokyle/kludge/common"
import "log"

var reqQ chan *common.Request

func startPool() {
	reqQ = make(chan *common.Request, reqBuf)
	for i := 0; i < poolSize; i++ {
		go requestHandler(i)
	}
}

func requestHandler(id int) {
	for {
		req, ok := <-reqQ
		if !ok {
			log.Printf("worker %d returns", id)
			return
		}
		req.Op.WID = id

		log.Printf("worker %d handling %s request", id,
			req.OpName())
		switch req.Op.OpCode {
		case common.OpGet:
			req.Resp <- store_get(req.Op)
		case common.OpSet:
			req.Resp <- store_set(req.Op)
		case common.OpDel:
			req.Resp <- store_del(req.Op)
		default:
			log.Printf("worker %d received invalid operation %d",
				id, req.Op.OpCode)
		}
	}
}

func store_get(op *common.Operation) (resp *common.Response) {
	ropts := levigo.NewReadOptions()
	ropts.SetVerifyChecksums(true)

	data, err := ldb.Get(ropts, op.Key)
	if err != nil {
		log.Printf("error handling get from worker %d: %s",
			op.WID, err.Error())
	} else {
		log.Printf("worker %d successfully completes GET", op.WID)
		resp = new(common.Response)
		resp.Body = data
	}
	return
}

func store_set(op *common.Operation) (resp *common.Response) {
	ropts := levigo.NewReadOptions()
	ropts.SetVerifyChecksums(true)

	data, err := ldb.Get(ropts, op.Key)
	if err != nil {
		log.Printf("worker %d failed to read key: %s", op.WID,
			err.Error())
		return
	}

	wopts := levigo.NewWriteOptions()
	wopts.SetSync(true)
	err = ldb.Put(wopts, op.Key, op.Val)
	if err != nil {
		log.Printf("worker %d failed to set key: %s", op.WID,
			err.Error())
		return
	} else {
		log.Printf("worker %d successfully wrote key", op.WID)
		resp = new(common.Response)
		resp.Body = data
	}
	return
}

func store_del(op *common.Operation) (resp *common.Response) {
	ropts := levigo.NewReadOptions()
	ropts.SetVerifyChecksums(true)

	data, err := ldb.Get(ropts, op.Key)
	if err != nil {
		log.Printf("worker %d failed to read key: %s", op.WID,
			err.Error())
		return
	}

	wopts := levigo.NewWriteOptions()
	wopts.SetSync(true)
	err = ldb.Delete(wopts, op.Key)
	if err != nil {
		log.Printf("worker %d failed to delete key: %s", op.WID,
			err.Error())
		return
	} else {
		if len(data) > 0 {
			resp.Body = []byte("ok")
		}
	}
	return
}

func store_lst(op *common.Operation) (resp *common.Response) {
	keys := make([]string, 0)
	ro := levigo.NewReadOptions()
	ro.SetFillCache(false)
	it := ldb.NewIterator(ro)
	for it = it; it.Valid(); it.Next() {
		keys = append(keys, string(it.Key()))
	}

	if err := it.GetError(); err != nil {
		log.Printf("worker %d failed to iterate over keys: %s",
			op.WID, err.Error())
	} else {
		resp = new(common.Response)
		resp.Body, err = json.Marshal(keys)
		if err != nil {
			log.Printf("worker %d failed to create JSON response: %s",
				op.WID, err.Error())
			resp.Body = []byte{}
		}
	}
	return
}
<|endoftext|>


<commit_before>package encoding

import (
	"encoding/binary"
	"io"
)

// Write writes an item to the Writer.
func Write(w io.Writer, item interface{}) (int64, error) {
	if wt, ok := item.(io.WriterTo); ok {
		return wt.WriteTo(w)
	}

	err := binary.Write(w, binary.BigEndian, item)
	if err != nil {
		return 0, err
	}

	return int64(binary.Size(item)), nil
}

// WriteSome writes multiple items to the Writer.
func WriteSome(w io.Writer, items ...interface{}) (n int64, err error) {
	for _, item := range items {
		len, err := Write(w, item)
		n += len
		if err != nil {
			return n, err
		}
	}

	return
}

// Read reads an item from the Reader.
func Read(r io.Reader, item interface{}) (int64, error) {
	if rf, ok := item.(io.ReaderFrom); ok {
		return rf.ReadFrom(r)
	}

	err := binary.Read(r, binary.BigEndian, item)
	if err != nil {
		return 0, err
	}

	return int64(binary.Size(item)), nil
}

// ReadSome reads multiple items from the Reader.
func ReadSome(r io.Reader, items ...interface{}) (n int64, err error) {
	for item := range items {
		len, err := Read(r, item)
		n += len
		if err != nil {
			return n, err
		}
	}

	return
}
<commit_msg>Fix ReadSome<commit_after>package encoding

import (
	"encoding/binary"
	"io"
)

// Write writes an item to the Writer.
func Write(w io.Writer, item interface{}) (int64, error) {
	if wt, ok := item.(io.WriterTo); ok {
		return wt.WriteTo(w)
	}

	err := binary.Write(w, binary.BigEndian, item)
	if err != nil {
		return 0, err
	}

	return int64(binary.Size(item)), nil
}

// WriteSome writes multiple items to the Writer.
func WriteSome(w io.Writer, items ...interface{}) (n int64, err error) {
	for _, item := range items {
		len, err := Write(w, item)
		n += len
		if err != nil {
			return n, err
		}
	}

	return
}

// Read reads an item from the Reader.
func Read(r io.Reader, item interface{}) (int64, error) {
	if rf, ok := item.(io.ReaderFrom); ok {
		return rf.ReadFrom(r)
	}

	err := binary.Read(r, binary.BigEndian, item)
	if err != nil {
		return 0, err
	}

	return int64(binary.Size(item)), nil
}

// ReadSome reads multiple items from the Reader.
func ReadSome(r io.Reader, items ...interface{}) (n int64, err error) {
	for _, item := range items {
		len, err := Read(r, item)
		n += len
		if err != nil {
			return n, err
		}
	}

	return
}
<|endoftext|>


<commit_before>package bolt

import (
	"context"
	"fmt"
	"os"
	"time"

	"github.com/coreos/bbolt"
	"github.com/influxdata/platform"
	"github.com/influxdata/platform/rand"
	"github.com/influxdata/platform/snowflake"
	"go.uber.org/zap"
)

const (
	// ErrUnableToOpen means we had an issue establishing a connection (or creating the database)
	ErrUnableToOpen = "Unable to open boltdb; is there a chronograf already running?  %v"
	// ErrUnableToBackup means we couldn't copy the db file into ./backup
	ErrUnableToBackup = "Unable to backup your database prior to migrations:  %v"
	// ErrUnableToInitialize means we couldn't create missing Buckets (maybe a timeout)
	ErrUnableToInitialize = "Unable to boot boltdb:  %v"
	// ErrUnableToMigrate means we had an issue changing the db schema
	ErrUnableToMigrate = "Unable to migrate boltdb:  %v"
)

// Client is a client for the boltDB data store.
type Client struct {
	Path   string
	db     *bolt.DB
	Logger *zap.Logger

	IDGenerator    platform.IDGenerator
	TokenGenerator platform.TokenGenerator
	time           func() time.Time
}

// NewClient returns an instance of a Client.
func NewClient() *Client {
	return &Client{
		Logger:         zap.NewNop(),
		IDGenerator:    snowflake.NewIDGenerator(),
		TokenGenerator: rand.NewTokenGenerator(64),
		time:           time.Now,
	}
}

// DB returns the clients DB.
func (c *Client) DB() *bolt.DB {
	return c.db
}

// WithLogger sets the logger an a client. It should not be called after
// the client has been open.
func (c *Client) WithLogger(l *zap.Logger) {
	c.Logger = l
}

// WithTime sets the function for computing the current time. Used for updating meta data
// about objects stored. Should only be used in tests for mocking.
func (c *Client) WithTime(fn func() time.Time) {
	c.time = fn
}

// Open / create boltDB file.
func (c *Client) Open(ctx context.Context) error {
	if _, err := os.Stat(c.Path); err != nil && !os.IsNotExist(err) {
		return err
	}

	// Open database file.
	db, err := bolt.Open(c.Path, 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		return fmt.Errorf(ErrUnableToOpen, err)
	}
	c.db = db

	if err := c.initialize(ctx); err != nil {
		return err
	}

	c.Logger.Info("Opened bolt database", zap.String("path", c.Path))
	return nil
}

// initialize creates Buckets that are missing
func (c *Client) initialize(ctx context.Context) error {
	if err := c.db.Update(func(tx *bolt.Tx) error {
		// Always create Buckets bucket.
		if err := c.initializeBuckets(ctx, tx); err != nil {
			return err
		}

		// Always create Organizations bucket.
		if err := c.initializeOrganizations(ctx, tx); err != nil {
			return err
		}

		// Always create Dashboards bucket.
		if err := c.initializeDashboards(ctx, tx); err != nil {
			return err
		}

		// Always create User bucket.
		if err := c.initializeUsers(ctx, tx); err != nil {
			return err
		}

		// Always create Authorization bucket.
		if err := c.initializeAuthorizations(ctx, tx); err != nil {
			return err
		}

		// Always create Onboarding bucket.
		if err := c.initializeOnboarding(ctx, tx); err != nil {
			return err
		}

		// Always create Telegraf Config bucket.
		if err := c.initializeTelegraf(ctx, tx); err != nil {
			return err
		}

		// Always create Source bucket.
		if err := c.initializeSources(ctx, tx); err != nil {
			return err
		}

		// Always create Views bucket.
		if err := c.initializeViews(ctx, tx); err != nil {
			return err
		}

		// Always create Macros bucket.
		if err := c.initializeMacros(ctx, tx); err != nil {
			return err
		}

		// Always create Scraper bucket.
		if err := c.initializeScraperTargets(ctx, tx); err != nil {
			return err
		}

		// Always create UserResourceMapping bucket.
		if err := c.initializeUserResourceMappings(ctx, tx); err != nil {
			return err
		}

		// Always create Session bucket.
		if err := c.initializeSessions(ctx, tx); err != nil {
			return err
		}

		return nil
	}); err != nil {
		return err
	}

	return nil
}

// Close the connection to the bolt database
func (c *Client) Close() error {
	if c.db != nil {
		return c.db.Close()
	}
	return nil
}
<commit_msg>fix(bolt): create bolt/data dir if not exist<commit_after>package bolt

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/coreos/bbolt"
	"github.com/influxdata/platform"
	"github.com/influxdata/platform/rand"
	"github.com/influxdata/platform/snowflake"
	"go.uber.org/zap"
)

const (
	// ErrUnableToOpen means we had an issue establishing a connection (or creating the database)
	ErrUnableToOpen = "Unable to open boltdb; is there a chronograf already running?  %v"
	// ErrUnableToBackup means we couldn't copy the db file into ./backup
	ErrUnableToBackup = "Unable to backup your database prior to migrations:  %v"
	// ErrUnableToInitialize means we couldn't create missing Buckets (maybe a timeout)
	ErrUnableToInitialize = "Unable to boot boltdb:  %v"
	// ErrUnableToMigrate means we had an issue changing the db schema
	ErrUnableToMigrate = "Unable to migrate boltdb:  %v"
)

// Client is a client for the boltDB data store.
type Client struct {
	Path   string
	db     *bolt.DB
	Logger *zap.Logger

	IDGenerator    platform.IDGenerator
	TokenGenerator platform.TokenGenerator
	time           func() time.Time
}

// NewClient returns an instance of a Client.
func NewClient() *Client {
	return &Client{
		Logger:         zap.NewNop(),
		IDGenerator:    snowflake.NewIDGenerator(),
		TokenGenerator: rand.NewTokenGenerator(64),
		time:           time.Now,
	}
}

// DB returns the clients DB.
func (c *Client) DB() *bolt.DB {
	return c.db
}

// WithLogger sets the logger an a client. It should not be called after
// the client has been open.
func (c *Client) WithLogger(l *zap.Logger) {
	c.Logger = l
}

// WithTime sets the function for computing the current time. Used for updating meta data
// about objects stored. Should only be used in tests for mocking.
func (c *Client) WithTime(fn func() time.Time) {
	c.time = fn
}

// Open / create boltDB file.
func (c *Client) Open(ctx context.Context) error {
	// Ensure the required directory structure exists.
	if err := os.MkdirAll(filepath.Dir(c.Path), 0700); err != nil {
		return fmt.Errorf("unable to create directory %s: %v", c.Path, err)
	}

	if _, err := os.Stat(c.Path); err != nil && !os.IsNotExist(err) {
		return err
	}

	// Open database file.
	db, err := bolt.Open(c.Path, 0600, &bolt.Options{Timeout: 1 * time.Second})
	if err != nil {
		return fmt.Errorf(ErrUnableToOpen, err)
	}
	c.db = db

	if err := c.initialize(ctx); err != nil {
		return err
	}

	c.Logger.Info("Opened bolt database", zap.String("path", c.Path))
	return nil
}

// initialize creates Buckets that are missing
func (c *Client) initialize(ctx context.Context) error {
	if err := c.db.Update(func(tx *bolt.Tx) error {
		// Always create Buckets bucket.
		if err := c.initializeBuckets(ctx, tx); err != nil {
			return err
		}

		// Always create Organizations bucket.
		if err := c.initializeOrganizations(ctx, tx); err != nil {
			return err
		}

		// Always create Dashboards bucket.
		if err := c.initializeDashboards(ctx, tx); err != nil {
			return err
		}

		// Always create User bucket.
		if err := c.initializeUsers(ctx, tx); err != nil {
			return err
		}

		// Always create Authorization bucket.
		if err := c.initializeAuthorizations(ctx, tx); err != nil {
			return err
		}

		// Always create Onboarding bucket.
		if err := c.initializeOnboarding(ctx, tx); err != nil {
			return err
		}

		// Always create Telegraf Config bucket.
		if err := c.initializeTelegraf(ctx, tx); err != nil {
			return err
		}

		// Always create Source bucket.
		if err := c.initializeSources(ctx, tx); err != nil {
			return err
		}

		// Always create Views bucket.
		if err := c.initializeViews(ctx, tx); err != nil {
			return err
		}

		// Always create Macros bucket.
		if err := c.initializeMacros(ctx, tx); err != nil {
			return err
		}

		// Always create Scraper bucket.
		if err := c.initializeScraperTargets(ctx, tx); err != nil {
			return err
		}

		// Always create UserResourceMapping bucket.
		if err := c.initializeUserResourceMappings(ctx, tx); err != nil {
			return err
		}

		// Always create Session bucket.
		if err := c.initializeSessions(ctx, tx); err != nil {
			return err
		}

		return nil
	}); err != nil {
		return err
	}

	return nil
}

// Close the connection to the bolt database
func (c *Client) Close() error {
	if c.db != nil {
		return c.db.Close()
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package boltdbpool implements a pool container for BoltDB github.com/coreos/bbolt databases.
Pool elements called connections keep reference counts for each database to close it
when it when the count is 0. Database is reused or opened based on database file path. Closing
the database must not be done directly, instead Connection.Close() method should be used.
Database is removed form the pool and closed by the goroutine in the background in respect to
reference count and delay in time if it is specified.

Example:

    package main

    import (
        "fmt"
        "time"

        "resenje.org/boltdbpool"
    )

    func main() {
        pool := boltdbpool.New(&boltdbpool.Options{
            ConnectionExpires: 5 * time.Second,
            ErrorHandler: boltdbpool.ErrorHandlerFunc(func(err error) {
                fmt.Printf("error: %v", err)
            }),
        })
        defer p.Close()

        ...

        c, err := pool.Get("/tmp/db.bolt")
        if err != nil {
            panic(err)
        }
        defer c.Close()

        ...

        c.DB.Update(func(tx *bolt.TX) error {
            ...
        })
    }
*/
package boltdbpool // import "resenje.org/boltdbpool"

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sync"
	"time"

	bolt "go.etcd.io/bbolt"
)

var (
	// DefaultFileMode is used in bolt.Open() as file mode for database file
	// if FileMode is not specified in boltdbpool.Options.
	DefaultFileMode = os.FileMode(0640)

	// DefaultDirMode is used in os.MkdirAll() as file mode for database directories
	// if DirMode is not specified in boltdbpool.Options.
	DefaultDirMode = os.FileMode(0750)

	// DefaultErrorHandler accepts errors from
	// goroutine that closes the databases if ErrorHandler is not specified in
	// boltdbpool.Options.
	DefaultErrorHandler = ErrorHandlerFunc(func(err error) {
		log.Printf("error: %v", err)
	})
)

// Connection encapsulates bolt.DB and keeps reference counter and closing time information.
type Connection struct {
	DB *bolt.DB

	pool      *Pool
	path      string
	count     int64
	closeTime time.Time
	mu        sync.RWMutex
}

// Close function on Connection decrements reference counter and closes the database if needed.
func (c *Connection) Close() {
	c.decrement()
	if c.count <= 0 {
		if c.pool.options.ConnectionExpires == 0 {
			c.pool.mu.Lock()
			c.pool.handleError(c.removeFromPool())
			c.pool.mu.Unlock()
			return
		}

		c.mu.Lock()
		c.closeTime = time.Now().Add(c.pool.options.ConnectionExpires)
		c.mu.Unlock()
		select {
		case c.pool.removeTrigger <- struct{}{}:
		default:
		}
	}
}

func (c *Connection) increment() {
	c.mu.Lock()

	// Reset the closing time
	c.closeTime = time.Time{}
	c.count++

	c.mu.Unlock()
}

func (c *Connection) decrement() {
	c.mu.Lock()

	c.count--

	c.mu.Unlock()
}

func (c *Connection) removeFromPool() error {
	return c.pool.remove(c.path)
}

// ErrorHandler interface can be used for objects that log or panic on error
type ErrorHandler interface {
	HandleError(err error)
}

// The ErrorHandlerFunc type is an adapter to allow the use of
// ordinary functions as error handlers.
type ErrorHandlerFunc func(err error)

// HandleError calls f(err).
func (f ErrorHandlerFunc) HandleError(err error) {
	f(err)
}

// Options are used when a new pool is created that.
type Options struct {
	// BoltOptions is used on bolt.Open().
	BoltOptions *bolt.Options

	// FileMode is used in bolt.Open() as file mode for database file. Default: 0640.
	FileMode os.FileMode

	// DirMode is used in os.MkdirAll() as file mode for database directories. Default: 0750.
	DirMode os.FileMode

	// ConnectionExpires is a duration between the reference count drops to 0 and
	// the time when the database is closed. It is useful to avoid frequent
	// openings of the same database. If the value is 0 (default), no caching is done.
	ConnectionExpires time.Duration

	// ErrorHandler represents interface that accepts errors from goroutine that closes the databases.
	ErrorHandler ErrorHandler
}

// Pool keeps track of connections.
type Pool struct {
	options       *Options
	connections   map[string]*Connection
	mu            sync.RWMutex
	removeTrigger chan struct{}
	quit          chan struct{}
}

// New creates new pool with provided options and also starts database closing goroutone
// and goroutine for errors handling to ErrorHandler.
func New(options *Options) *Pool {
	if options == nil {
		options = &Options{}
	}
	if options.FileMode == 0 {
		options.FileMode = DefaultFileMode
	}
	if options.DirMode == 0 {
		options.DirMode = DefaultDirMode
	}
	if options.ErrorHandler == nil {
		options.ErrorHandler = DefaultErrorHandler
	}
	p := &Pool{
		options:       options,
		connections:   map[string]*Connection{},
		removeTrigger: make(chan struct{}, 1),
		quit:          make(chan struct{}),
	}
	go func() {
		for {
			select {
			case <-p.removeTrigger:
				select {
				case <-time.After(p.options.ConnectionExpires):
				case <-p.quit:
					return
				}
				p.mu.Lock()
				for _, c := range p.connections {
					c.mu.RLock()
					if !c.closeTime.IsZero() && c.closeTime.Before(time.Now()) {
						p.handleError(c.removeFromPool())
					}
					c.mu.RUnlock()
				}
				p.mu.Unlock()
			case <-p.quit:
				return
			}
		}
	}()
	return p
}

// Get returns a connection that contains a database or creates a new connection
// with newly opened database based on options specified on pool creation.
func (p *Pool) Get(path string) (*Connection, error) {
	p.mu.Lock()
	defer p.mu.Unlock()

	if c, ok := p.connections[path]; ok {
		c.increment()
		return c, nil
	}
	if _, err := os.Stat(filepath.Dir(path)); os.IsNotExist(err) {
		if err := os.MkdirAll(filepath.Dir(path), p.options.DirMode); err != nil {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	}
	db, err := bolt.Open(path, p.options.FileMode, p.options.BoltOptions)
	if err != nil {
		return nil, err
	}
	c := &Connection{
		DB:   db,
		path: path,
		pool: p,
	}
	p.connections[path] = c

	c.increment()
	return c, nil
}

// Has returns true if a database with a file path is in the pool.
func (p *Pool) Has(path string) bool {
	p.mu.RLock()
	defer p.mu.RUnlock()

	_, ok := p.connections[path]
	return ok
}

// Close function closes and removes from the pool all databases. After the execution
// pool is not usable.
func (p *Pool) Close() {
	p.mu.Lock()
	defer p.mu.Unlock()

	for _, c := range p.connections {
		p.handleError(c.removeFromPool())
	}
	close(p.quit)
}

func (p *Pool) remove(path string) error {
	c, ok := p.connections[path]
	if !ok {
		return fmt.Errorf("boltdbpool: Unknown DB %s", path)
	}
	delete(p.connections, path)
	return c.DB.Close()
}

func (p *Pool) handleError(err error) {
	if err != nil {
		p.options.ErrorHandler.HandleError(err)
	}
}
<commit_msg>More permissive DefaultFileMode and DefaultDirMode mask values<commit_after>// Copyright (c) 2015 Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package boltdbpool implements a pool container for BoltDB github.com/coreos/bbolt databases.
Pool elements called connections keep reference counts for each database to close it
when it when the count is 0. Database is reused or opened based on database file path. Closing
the database must not be done directly, instead Connection.Close() method should be used.
Database is removed form the pool and closed by the goroutine in the background in respect to
reference count and delay in time if it is specified.

Example:

    package main

    import (
        "fmt"
        "time"

        "resenje.org/boltdbpool"
    )

    func main() {
        pool := boltdbpool.New(&boltdbpool.Options{
            ConnectionExpires: 5 * time.Second,
            ErrorHandler: boltdbpool.ErrorHandlerFunc(func(err error) {
                fmt.Printf("error: %v", err)
            }),
        })
        defer p.Close()

        ...

        c, err := pool.Get("/tmp/db.bolt")
        if err != nil {
            panic(err)
        }
        defer c.Close()

        ...

        c.DB.Update(func(tx *bolt.TX) error {
            ...
        })
    }
*/
package boltdbpool // import "resenje.org/boltdbpool"

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sync"
	"time"

	bolt "go.etcd.io/bbolt"
)

var (
	// DefaultFileMode is used in bolt.Open() as file mode for database file
	// if FileMode is not specified in boltdbpool.Options.
	DefaultFileMode = os.FileMode(0666)

	// DefaultDirMode is used in os.MkdirAll() as file mode for database directories
	// if DirMode is not specified in boltdbpool.Options.
	DefaultDirMode = os.FileMode(0777)

	// DefaultErrorHandler accepts errors from
	// goroutine that closes the databases if ErrorHandler is not specified in
	// boltdbpool.Options.
	DefaultErrorHandler = ErrorHandlerFunc(func(err error) {
		log.Printf("error: %v", err)
	})
)

// Connection encapsulates bolt.DB and keeps reference counter and closing time information.
type Connection struct {
	DB *bolt.DB

	pool      *Pool
	path      string
	count     int64
	closeTime time.Time
	mu        sync.RWMutex
}

// Close function on Connection decrements reference counter and closes the database if needed.
func (c *Connection) Close() {
	c.decrement()
	if c.count <= 0 {
		if c.pool.options.ConnectionExpires == 0 {
			c.pool.mu.Lock()
			c.pool.handleError(c.removeFromPool())
			c.pool.mu.Unlock()
			return
		}

		c.mu.Lock()
		c.closeTime = time.Now().Add(c.pool.options.ConnectionExpires)
		c.mu.Unlock()
		select {
		case c.pool.removeTrigger <- struct{}{}:
		default:
		}
	}
}

func (c *Connection) increment() {
	c.mu.Lock()

	// Reset the closing time
	c.closeTime = time.Time{}
	c.count++

	c.mu.Unlock()
}

func (c *Connection) decrement() {
	c.mu.Lock()

	c.count--

	c.mu.Unlock()
}

func (c *Connection) removeFromPool() error {
	return c.pool.remove(c.path)
}

// ErrorHandler interface can be used for objects that log or panic on error
type ErrorHandler interface {
	HandleError(err error)
}

// The ErrorHandlerFunc type is an adapter to allow the use of
// ordinary functions as error handlers.
type ErrorHandlerFunc func(err error)

// HandleError calls f(err).
func (f ErrorHandlerFunc) HandleError(err error) {
	f(err)
}

// Options are used when a new pool is created that.
type Options struct {
	// BoltOptions is used on bolt.Open().
	BoltOptions *bolt.Options

	// FileMode is used in bolt.Open() as file mode for database file. Default: 0640.
	FileMode os.FileMode

	// DirMode is used in os.MkdirAll() as file mode for database directories. Default: 0750.
	DirMode os.FileMode

	// ConnectionExpires is a duration between the reference count drops to 0 and
	// the time when the database is closed. It is useful to avoid frequent
	// openings of the same database. If the value is 0 (default), no caching is done.
	ConnectionExpires time.Duration

	// ErrorHandler represents interface that accepts errors from goroutine that closes the databases.
	ErrorHandler ErrorHandler
}

// Pool keeps track of connections.
type Pool struct {
	options       *Options
	connections   map[string]*Connection
	mu            sync.RWMutex
	removeTrigger chan struct{}
	quit          chan struct{}
}

// New creates new pool with provided options and also starts database closing goroutone
// and goroutine for errors handling to ErrorHandler.
func New(options *Options) *Pool {
	if options == nil {
		options = &Options{}
	}
	if options.FileMode == 0 {
		options.FileMode = DefaultFileMode
	}
	if options.DirMode == 0 {
		options.DirMode = DefaultDirMode
	}
	if options.ErrorHandler == nil {
		options.ErrorHandler = DefaultErrorHandler
	}
	p := &Pool{
		options:       options,
		connections:   map[string]*Connection{},
		removeTrigger: make(chan struct{}, 1),
		quit:          make(chan struct{}),
	}
	go func() {
		for {
			select {
			case <-p.removeTrigger:
				select {
				case <-time.After(p.options.ConnectionExpires):
				case <-p.quit:
					return
				}
				p.mu.Lock()
				for _, c := range p.connections {
					c.mu.RLock()
					if !c.closeTime.IsZero() && c.closeTime.Before(time.Now()) {
						p.handleError(c.removeFromPool())
					}
					c.mu.RUnlock()
				}
				p.mu.Unlock()
			case <-p.quit:
				return
			}
		}
	}()
	return p
}

// Get returns a connection that contains a database or creates a new connection
// with newly opened database based on options specified on pool creation.
func (p *Pool) Get(path string) (*Connection, error) {
	p.mu.Lock()
	defer p.mu.Unlock()

	if c, ok := p.connections[path]; ok {
		c.increment()
		return c, nil
	}
	if _, err := os.Stat(filepath.Dir(path)); os.IsNotExist(err) {
		if err := os.MkdirAll(filepath.Dir(path), p.options.DirMode); err != nil {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	}
	db, err := bolt.Open(path, p.options.FileMode, p.options.BoltOptions)
	if err != nil {
		return nil, err
	}
	c := &Connection{
		DB:   db,
		path: path,
		pool: p,
	}
	p.connections[path] = c

	c.increment()
	return c, nil
}

// Has returns true if a database with a file path is in the pool.
func (p *Pool) Has(path string) bool {
	p.mu.RLock()
	defer p.mu.RUnlock()

	_, ok := p.connections[path]
	return ok
}

// Close function closes and removes from the pool all databases. After the execution
// pool is not usable.
func (p *Pool) Close() {
	p.mu.Lock()
	defer p.mu.Unlock()

	for _, c := range p.connections {
		p.handleError(c.removeFromPool())
	}
	close(p.quit)
}

func (p *Pool) remove(path string) error {
	c, ok := p.connections[path]
	if !ok {
		return fmt.Errorf("boltdbpool: Unknown DB %s", path)
	}
	delete(p.connections, path)
	return c.DB.Close()
}

func (p *Pool) handleError(err error) {
	if err != nil {
		p.options.ErrorHandler.HandleError(err)
	}
}
<|endoftext|>


<commit_before>package glib

// #cgo pkg-config: glib-2.0 gobject-2.0
// #include <gio/gio.h>
// #include <glib.h>
// #include <glib-object.h>
// #include "glib.go.h"
import "C"
import "unsafe"

// Settings is a representation of GSettings.
type Settings struct {
	*Object
}

// native() returns a pointer to the underlying GSettings.
func (v *Settings) native() *C.GSettings {
	if v == nil || v.GObject == nil {
		return nil
	}
	return C.toGSettings(unsafe.Pointer(v.GObject))
}

func (v *Settings) Native() uintptr {
	return uintptr(unsafe.Pointer(v.native()))
}

func marshalSettings(p uintptr) (interface{}, error) {
	c := C.g_value_get_object((*C.GValue)(unsafe.Pointer(p)))
	return wrapSettings(wrapObject(unsafe.Pointer(c))), nil
}

func wrapSettings(obj *Object) *Settings {
	return &Settings{obj}
}

func wrapFullSettings(obj *C.GSettings) *Settings {
	if obj == nil {
		return nil
	}
	return wrapSettings(wrapObject(unsafe.Pointer(obj)))
}

// SettingsNew is a wrapper around g_settings_new().
func SettingsNew(schemaID string) *Settings {
	cstr := (*C.gchar)(C.CString(schemaID))
	defer C.free(unsafe.Pointer(cstr))

	return wrapFullSettings(C.g_settings_new(cstr))
}

// SettingsNewWithPath is a wrapper around g_settings_new_with_path().
func SettingsNewWithPath(schemaID, path string) *Settings {
	cstr1 := (*C.gchar)(C.CString(schemaID))
	defer C.free(unsafe.Pointer(cstr1))

	cstr2 := (*C.gchar)(C.CString(path))
	defer C.free(unsafe.Pointer(cstr2))

	return wrapFullSettings(C.g_settings_new_with_path(cstr1, cstr2))
}

// SettingsNewWithBackend is a wrapper around g_settings_new_with_backend().
func SettingsNewWithBackend(schemaID string, backend *SettingsBackend) *Settings {
	cstr1 := (*C.gchar)(C.CString(schemaID))
	defer C.free(unsafe.Pointer(cstr1))

	return wrapFullSettings(C.g_settings_new_with_backend(cstr1, backend.native()))
}

// SettingsNewWithBackendAndPath is a wrapper around g_settings_new_with_backend_and_path().
func SettingsNewWithBackendAndPath(schemaID string, backend *SettingsBackend, path string) *Settings {
	cstr1 := (*C.gchar)(C.CString(schemaID))
	defer C.free(unsafe.Pointer(cstr1))

	cstr2 := (*C.gchar)(C.CString(path))
	defer C.free(unsafe.Pointer(cstr2))

	return wrapFullSettings(C.g_settings_new_with_backend_and_path(cstr1, backend.native(), cstr2))
}

// SettingsNewFull is a wrapper around g_settings_new_full().
func SettingsNewFull(schema *SettingsSchema, backend *SettingsBackend, path string) *Settings {
	cstr1 := (*C.gchar)(C.CString(path))
	defer C.free(unsafe.Pointer(cstr1))

	return wrapFullSettings(C.g_settings_new_full(schema.native(), backend.native(), cstr1))
}

// SettingsSync is a wrapper around g_settings_sync().
func SettingsSync() {
	C.g_settings_sync()
}

// IsWritable is a wrapper around g_settings_is_writable().
func (v *Settings) IsWritable(name string) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_is_writable(v.native(), cstr1))
}

// Delay is a wrapper around g_settings_delay().
func (v *Settings) Delay() {
	C.g_settings_delay(v.native())
}

// Apply is a wrapper around g_settings_apply().
func (v *Settings) Apply() {
	C.g_settings_apply(v.native())
}

// Revert is a wrapper around g_settings_revert().
func (v *Settings) Revert() {
	C.g_settings_revert(v.native())
}

// GetHasUnapplied is a wrapper around g_settings_get_has_unapplied().
func (v *Settings) GetHasUnapplied() bool {
	return gobool(C.g_settings_get_has_unapplied(v.native()))
}

// GetChild is a wrapper around g_settings_get_child().
func (v *Settings) GetChild(name string) *Settings {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return wrapFullSettings(C.g_settings_get_child(v.native(), cstr1))
}

// Reset is a wrapper around g_settings_reset().
func (v *Settings) Reset(name string) {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	C.g_settings_reset(v.native(), cstr1)
}

// ListChildren is a wrapper around g_settings_list_children().
func (v *Settings) ListChildren() []string {
	return toGoStringArray(C.g_settings_list_children(v.native()))
}

// GetBoolean is a wrapper around g_settings_get_boolean().
func (v *Settings) GetBoolean(name string) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_get_boolean(v.native(), cstr1))
}

// SetBoolean is a wrapper around g_settings_set_boolean().
func (v *Settings) SetBoolean(name string, value bool) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_set_boolean(v.native(), cstr1, gbool(value)))
}

// GetInt is a wrapper around g_settings_get_int().
func (v *Settings) GetInt(name string) int {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return int(C.g_settings_get_int(v.native(), cstr1))
}

// SetInt is a wrapper around g_settings_set_int().
func (v *Settings) SetInt(name string, value int) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_set_int(v.native(), cstr1, C.gint(value)))
}

// GetUInt is a wrapper around g_settings_get_uint().
func (v *Settings) GetUInt(name string) uint {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return uint(C.g_settings_get_uint(v.native(), cstr1))
}

// SetUInt is a wrapper around g_settings_set_uint().
func (v *Settings) SetUInt(name string, value uint) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_set_uint(v.native(), cstr1, C.guint(value)))
}

// GetDouble is a wrapper around g_settings_get_double().
func (v *Settings) GetDouble(name string) float64 {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return float64(C.g_settings_get_double(v.native(), cstr1))
}

// SetDouble is a wrapper around g_settings_set_double().
func (v *Settings) SetDouble(name string, value float64) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_set_double(v.native(), cstr1, C.gdouble(value)))
}

// GetString is a wrapper around g_settings_get_string().
func (v *Settings) GetString(name string) string {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return C.GoString((*C.char)(C.g_settings_get_string(v.native(), cstr1)))
}

// SetString is a wrapper around g_settings_set_string().
func (v *Settings) SetString(name string, value string) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	cstr2 := (*C.gchar)(C.CString(value))
	defer C.free(unsafe.Pointer(cstr2))

	return gobool(C.g_settings_set_string(v.native(), cstr1, cstr2))
}

// GetEnum is a wrapper around g_settings_get_enum().
func (v *Settings) GetEnum(name string) int {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return int(C.g_settings_get_enum(v.native(), cstr1))
}

// GetStrv is a wrapper around g_settings_get_strv().
func (v *Settings) GetStrv(name string) []string {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))
	return toGoStringArray(C.g_settings_get_strv(v.native(), cstr1))
}

// SetStrv is a wrapper around g_settings_set_strv().
func (v *Settings) SetStrv(name string, values []string) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	cvalues := C.make_strings(C.int(len(values) + 1))
	defer C.destroy_strings(cvalues)

	for i, accel := range values {
		cstr := C.CString(accel)
		defer C.free(unsafe.Pointer(cstr))
		C.set_string(cvalues, C.int(i), (*C.gchar)(cstr))
	}
	C.set_string(cvalues, C.int(len(values)), nil)
	return gobool(C.g_settings_set_strv(v.native(), cstr1, cvalues))
}

// SetEnum is a wrapper around g_settings_set_enum().
func (v *Settings) SetEnum(name string, value int) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_set_enum(v.native(), cstr1, C.gint(value)))
}

// GetFlags is a wrapper around g_settings_get_flags().
func (v *Settings) GetFlags(name string) uint {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return uint(C.g_settings_get_flags(v.native(), cstr1))
}

// SetFlags is a wrapper around g_settings_set_flags().
func (v *Settings) SetFlags(name string, value uint) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_set_flags(v.native(), cstr1, C.guint(value)))
}

func (v *Settings) GetValue(name string) *Variant {
	cstr := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr))
	return newVariant(C.g_settings_get_value(v.native(), cstr))
}

// GVariant * 	g_settings_get_value ()
// gboolean 	g_settings_set_value ()
// GVariant * 	g_settings_get_user_value ()
// GVariant * 	g_settings_get_default_value ()
// const gchar * const * 	g_settings_list_schemas ()
// const gchar * const * 	g_settings_list_relocatable_schemas ()
// gchar ** 	g_settings_list_keys ()
// GVariant * 	g_settings_get_range ()
// gboolean 	g_settings_range_check ()
// void 	g_settings_get ()
// gboolean 	g_settings_set ()
// gpointer 	g_settings_get_mapped ()
// void 	g_settings_bind ()
// void 	g_settings_bind_with_mapping ()
// void 	g_settings_bind_writable ()
// void 	g_settings_unbind ()
// gaction * 	g_settings_create_action ()
// gchar ** 	g_settings_get_strv ()
// gboolean 	g_settings_set_strv ()
<commit_msg>Added SetStrv to glib.settings (fixed)<commit_after>package glib

// #cgo pkg-config: glib-2.0 gobject-2.0
// #include <gio/gio.h>
// #include <glib.h>
// #include <glib-object.h>
// #include "glib.go.h"
import "C"
import "unsafe"

// Settings is a representation of GSettings.
type Settings struct {
	*Object
}

// native() returns a pointer to the underlying GSettings.
func (v *Settings) native() *C.GSettings {
	if v == nil || v.GObject == nil {
		return nil
	}
	return C.toGSettings(unsafe.Pointer(v.GObject))
}

func (v *Settings) Native() uintptr {
	return uintptr(unsafe.Pointer(v.native()))
}

func marshalSettings(p uintptr) (interface{}, error) {
	c := C.g_value_get_object((*C.GValue)(unsafe.Pointer(p)))
	return wrapSettings(wrapObject(unsafe.Pointer(c))), nil
}

func wrapSettings(obj *Object) *Settings {
	return &Settings{obj}
}

func wrapFullSettings(obj *C.GSettings) *Settings {
	if obj == nil {
		return nil
	}
	return wrapSettings(wrapObject(unsafe.Pointer(obj)))
}

// SettingsNew is a wrapper around g_settings_new().
func SettingsNew(schemaID string) *Settings {
	cstr := (*C.gchar)(C.CString(schemaID))
	defer C.free(unsafe.Pointer(cstr))

	return wrapFullSettings(C.g_settings_new(cstr))
}

// SettingsNewWithPath is a wrapper around g_settings_new_with_path().
func SettingsNewWithPath(schemaID, path string) *Settings {
	cstr1 := (*C.gchar)(C.CString(schemaID))
	defer C.free(unsafe.Pointer(cstr1))

	cstr2 := (*C.gchar)(C.CString(path))
	defer C.free(unsafe.Pointer(cstr2))

	return wrapFullSettings(C.g_settings_new_with_path(cstr1, cstr2))
}

// SettingsNewWithBackend is a wrapper around g_settings_new_with_backend().
func SettingsNewWithBackend(schemaID string, backend *SettingsBackend) *Settings {
	cstr1 := (*C.gchar)(C.CString(schemaID))
	defer C.free(unsafe.Pointer(cstr1))

	return wrapFullSettings(C.g_settings_new_with_backend(cstr1, backend.native()))
}

// SettingsNewWithBackendAndPath is a wrapper around g_settings_new_with_backend_and_path().
func SettingsNewWithBackendAndPath(schemaID string, backend *SettingsBackend, path string) *Settings {
	cstr1 := (*C.gchar)(C.CString(schemaID))
	defer C.free(unsafe.Pointer(cstr1))

	cstr2 := (*C.gchar)(C.CString(path))
	defer C.free(unsafe.Pointer(cstr2))

	return wrapFullSettings(C.g_settings_new_with_backend_and_path(cstr1, backend.native(), cstr2))
}

// SettingsNewFull is a wrapper around g_settings_new_full().
func SettingsNewFull(schema *SettingsSchema, backend *SettingsBackend, path string) *Settings {
	cstr1 := (*C.gchar)(C.CString(path))
	defer C.free(unsafe.Pointer(cstr1))

	return wrapFullSettings(C.g_settings_new_full(schema.native(), backend.native(), cstr1))
}

// SettingsSync is a wrapper around g_settings_sync().
func SettingsSync() {
	C.g_settings_sync()
}

// IsWritable is a wrapper around g_settings_is_writable().
func (v *Settings) IsWritable(name string) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_is_writable(v.native(), cstr1))
}

// Delay is a wrapper around g_settings_delay().
func (v *Settings) Delay() {
	C.g_settings_delay(v.native())
}

// Apply is a wrapper around g_settings_apply().
func (v *Settings) Apply() {
	C.g_settings_apply(v.native())
}

// Revert is a wrapper around g_settings_revert().
func (v *Settings) Revert() {
	C.g_settings_revert(v.native())
}

// GetHasUnapplied is a wrapper around g_settings_get_has_unapplied().
func (v *Settings) GetHasUnapplied() bool {
	return gobool(C.g_settings_get_has_unapplied(v.native()))
}

// GetChild is a wrapper around g_settings_get_child().
func (v *Settings) GetChild(name string) *Settings {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return wrapFullSettings(C.g_settings_get_child(v.native(), cstr1))
}

// Reset is a wrapper around g_settings_reset().
func (v *Settings) Reset(name string) {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	C.g_settings_reset(v.native(), cstr1)
}

// ListChildren is a wrapper around g_settings_list_children().
func (v *Settings) ListChildren() []string {
	return toGoStringArray(C.g_settings_list_children(v.native()))
}

// GetBoolean is a wrapper around g_settings_get_boolean().
func (v *Settings) GetBoolean(name string) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_get_boolean(v.native(), cstr1))
}

// SetBoolean is a wrapper around g_settings_set_boolean().
func (v *Settings) SetBoolean(name string, value bool) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_set_boolean(v.native(), cstr1, gbool(value)))
}

// GetInt is a wrapper around g_settings_get_int().
func (v *Settings) GetInt(name string) int {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return int(C.g_settings_get_int(v.native(), cstr1))
}

// SetInt is a wrapper around g_settings_set_int().
func (v *Settings) SetInt(name string, value int) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_set_int(v.native(), cstr1, C.gint(value)))
}

// GetUInt is a wrapper around g_settings_get_uint().
func (v *Settings) GetUInt(name string) uint {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return uint(C.g_settings_get_uint(v.native(), cstr1))
}

// SetUInt is a wrapper around g_settings_set_uint().
func (v *Settings) SetUInt(name string, value uint) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_set_uint(v.native(), cstr1, C.guint(value)))
}

// GetDouble is a wrapper around g_settings_get_double().
func (v *Settings) GetDouble(name string) float64 {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return float64(C.g_settings_get_double(v.native(), cstr1))
}

// SetDouble is a wrapper around g_settings_set_double().
func (v *Settings) SetDouble(name string, value float64) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_set_double(v.native(), cstr1, C.gdouble(value)))
}

// GetString is a wrapper around g_settings_get_string().
func (v *Settings) GetString(name string) string {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return C.GoString((*C.char)(C.g_settings_get_string(v.native(), cstr1)))
}

// SetString is a wrapper around g_settings_set_string().
func (v *Settings) SetString(name string, value string) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	cstr2 := (*C.gchar)(C.CString(value))
	defer C.free(unsafe.Pointer(cstr2))

	return gobool(C.g_settings_set_string(v.native(), cstr1, cstr2))
}

// GetEnum is a wrapper around g_settings_get_enum().
func (v *Settings) GetEnum(name string) int {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return int(C.g_settings_get_enum(v.native(), cstr1))
}

// GetStrv is a wrapper around g_settings_get_strv().
func (v *Settings) GetStrv(name string) []string {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))
	return toGoStringArray(C.g_settings_get_strv(v.native(), cstr1))
}

// SetStrv is a wrapper around g_settings_set_strv().
func (v *Settings) SetStrv(name string, values []string) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	cvalues := make([]*C.gchar, len(values))
	for i, accel := range values {
		cvalues[i] = (*C.gchar)(C.CString(accel))
		defer C.free(unsafe.Pointer(cvalues[i]))
	}
	cvalues = append(cvalues, nil)

	return gobool(C.g_settings_set_strv(v.native(), cstr1, &cvalues[0]))
}

// SetEnum is a wrapper around g_settings_set_enum().
func (v *Settings) SetEnum(name string, value int) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_set_enum(v.native(), cstr1, C.gint(value)))
}

// GetFlags is a wrapper around g_settings_get_flags().
func (v *Settings) GetFlags(name string) uint {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return uint(C.g_settings_get_flags(v.native(), cstr1))
}

// SetFlags is a wrapper around g_settings_set_flags().
func (v *Settings) SetFlags(name string, value uint) bool {
	cstr1 := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr1))

	return gobool(C.g_settings_set_flags(v.native(), cstr1, C.guint(value)))
}

func (v *Settings) GetValue(name string) *Variant {
	cstr := (*C.gchar)(C.CString(name))
	defer C.free(unsafe.Pointer(cstr))
	return newVariant(C.g_settings_get_value(v.native(), cstr))
}

// GVariant * 	g_settings_get_value ()
// gboolean 	g_settings_set_value ()
// GVariant * 	g_settings_get_user_value ()
// GVariant * 	g_settings_get_default_value ()
// const gchar * const * 	g_settings_list_schemas ()
// const gchar * const * 	g_settings_list_relocatable_schemas ()
// gchar ** 	g_settings_list_keys ()
// GVariant * 	g_settings_get_range ()
// gboolean 	g_settings_range_check ()
// void 	g_settings_get ()
// gboolean 	g_settings_set ()
// gpointer 	g_settings_get_mapped ()
// void 	g_settings_bind ()
// void 	g_settings_bind_with_mapping ()
// void 	g_settings_bind_writable ()
// void 	g_settings_unbind ()
// gaction * 	g_settings_create_action ()
// gchar ** 	g_settings_get_strv ()
// gboolean 	g_settings_set_strv ()
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"sync"
	"time"

	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/metrics"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/shared/logger"
)

type metricsCacheEntry struct {
	metrics *metrics.MetricSet
	expiry  time.Time
}

var metricsCache map[string]metricsCacheEntry
var metricsCacheLock sync.Mutex
var metricsLock sync.Mutex

var metricsCmd = APIEndpoint{
	Path: "metrics",

	Get: APIEndpointAction{Handler: metricsGet, AccessHandler: allowProjectPermission("containers", "view")},
}

// swagger:operation GET /1.0/metrics metrics metrics_get
//
// Get metrics
//
// Gets metrics of instances.
//
// ---
// produces:
//   - text/plain
// parameters:
//   - in: query
//     name: project
//     description: Project name
//     type: string
//     example: default
// responses:
//   "200":
//     description: Metrics
//     schema:
//       type: string
//       description: Instance metrics
//   "403":
//     $ref: "#/responses/Forbidden"
//   "500":
//     $ref: "#/responses/InternalServerError"
func metricsGet(d *Daemon, r *http.Request) response.Response {
	projectName := queryParam(r, "project")

	// Figure out the projects to retrieve.
	var projectNames []string

	if projectName != "" {
		projectNames = []string{projectName}
	} else {
		// Get all projects.
		err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
			projects, err := tx.GetProjects(db.ProjectFilter{})
			if err != nil {
				return err
			}

			for _, project := range projects {
				projectNames = append(projectNames, project.Name)
			}

			return nil
		})
		if err != nil {
			return response.SmartError(err)
		}
	}

	// Prepare response.
	resp := metrics.NewMetricSet(nil)

	// Review the cache.
	metricsCacheLock.Lock()
	projectMissing := []string{}
	for _, project := range projectNames {
		cache, ok := metricsCache[project]
		if !ok || cache.expiry.Before(time.Now()) {
			// If missing or expired, record it.
			projectMissing = append(projectMissing, project)
			continue
		}

		// If present and valid, merge the existing data.
		resp.Merge(cache.metrics)
	}
	metricsCacheLock.Unlock()

	// If all valid, return immediately.
	if len(projectMissing) == 0 {
		return response.SyncResponsePlain(true, resp.String())
	}

	// Acquire update lock.
	metricsLock.Lock()
	defer metricsLock.Unlock()

	// Check if any of the missing data has been filled in.
	metricsCacheLock.Lock()
	toFetch := []string{}
	for _, project := range projectMissing {
		cache, ok := metricsCache[project]
		if !ok || cache.expiry.Before(time.Now()) {
			// Still missing, queue a re-fetch.
			toFetch = append(toFetch, project)
			continue
		}

		// If present and valid, merge the existing data.
		resp.Merge(cache.metrics)
	}
	metricsCacheLock.Unlock()

	// If all valid, return immediately.
	if len(toFetch) == 0 {
		return response.SyncResponsePlain(true, resp.String())
	}

	// Prepare temporary metrics storage.
	newMetrics := map[string]*metrics.MetricSet{}
	newMetricsLock := sync.Mutex{}

	// Fetch what's missing.
	wgInstances := sync.WaitGroup{}
	for _, project := range toFetch {
		newMetrics[project] = metrics.NewMetricSet(nil)

		// Get the instances.
		instances, err := instanceLoadNodeProjectAll(d.State(), project, instancetype.Any)
		if err != nil {
			return response.SmartError(err)
		}

		for _, inst := range instances {
			// Ignore stopped instances.
			if !inst.IsRunning() {
				continue
			}

			wgInstances.Add(1)
			go func(inst instance.Instance) {
				defer wgInstances.Done()

				instanceMetrics, err := inst.Metrics()
				if err != nil {
					logger.Warn("Failed to get instance metrics", log.Ctx{"instance": inst.Name(), "project": inst.Project(), "err": err})
					return
				}

				// Add the metrics.
				newMetricsLock.Lock()
				defer newMetricsLock.Unlock()

				newMetrics[inst.Project()].Merge(instanceMetrics)
			}(inst)
		}
	}

	wgInstances.Wait()

	// Put the new data in the global cache and in response.
	metricsCacheLock.Lock()

	if metricsCache == nil {
		metricsCache = map[string]metricsCacheEntry{}
	}

	for project, entries := range newMetrics {
		metricsCache[project] = metricsCacheEntry{
			expiry:  time.Now().Add(8 * time.Second),
			metrics: entries,
		}

		resp.Merge(entries)
	}
	metricsCacheLock.Unlock()

	return response.SyncResponsePlain(true, resp.String())
}
<commit_msg>lxd/api_metrics: Rename resp to metricSet<commit_after>package main

import (
	"net/http"
	"sync"
	"time"

	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/metrics"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/shared/logger"
)

type metricsCacheEntry struct {
	metrics *metrics.MetricSet
	expiry  time.Time
}

var metricsCache map[string]metricsCacheEntry
var metricsCacheLock sync.Mutex
var metricsLock sync.Mutex

var metricsCmd = APIEndpoint{
	Path: "metrics",

	Get: APIEndpointAction{Handler: metricsGet, AccessHandler: allowProjectPermission("containers", "view")},
}

// swagger:operation GET /1.0/metrics metrics metrics_get
//
// Get metrics
//
// Gets metrics of instances.
//
// ---
// produces:
//   - text/plain
// parameters:
//   - in: query
//     name: project
//     description: Project name
//     type: string
//     example: default
// responses:
//   "200":
//     description: Metrics
//     schema:
//       type: string
//       description: Instance metrics
//   "403":
//     $ref: "#/responses/Forbidden"
//   "500":
//     $ref: "#/responses/InternalServerError"
func metricsGet(d *Daemon, r *http.Request) response.Response {
	projectName := queryParam(r, "project")

	// Figure out the projects to retrieve.
	var projectNames []string

	if projectName != "" {
		projectNames = []string{projectName}
	} else {
		// Get all projects.
		err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
			projects, err := tx.GetProjects(db.ProjectFilter{})
			if err != nil {
				return err
			}

			for _, project := range projects {
				projectNames = append(projectNames, project.Name)
			}

			return nil
		})
		if err != nil {
			return response.SmartError(err)
		}
	}

	// Prepare response.
	metricSet := metrics.NewMetricSet(nil)

	// Review the cache.
	metricsCacheLock.Lock()
	projectMissing := []string{}
	for _, project := range projectNames {
		cache, ok := metricsCache[project]
		if !ok || cache.expiry.Before(time.Now()) {
			// If missing or expired, record it.
			projectMissing = append(projectMissing, project)
			continue
		}

		// If present and valid, merge the existing data.
		metricSet.Merge(cache.metrics)
	}
	metricsCacheLock.Unlock()

	// If all valid, return immediately.
	if len(projectMissing) == 0 {
		return response.SyncResponsePlain(true, metricSet.String())
	}

	// Acquire update lock.
	metricsLock.Lock()
	defer metricsLock.Unlock()

	// Check if any of the missing data has been filled in.
	metricsCacheLock.Lock()
	toFetch := []string{}
	for _, project := range projectMissing {
		cache, ok := metricsCache[project]
		if !ok || cache.expiry.Before(time.Now()) {
			// Still missing, queue a re-fetch.
			toFetch = append(toFetch, project)
			continue
		}

		// If present and valid, merge the existing data.
		metricSet.Merge(cache.metrics)
	}
	metricsCacheLock.Unlock()

	// If all valid, return immediately.
	if len(toFetch) == 0 {
		return response.SyncResponsePlain(true, metricSet.String())
	}

	// Prepare temporary metrics storage.
	newMetrics := map[string]*metrics.MetricSet{}
	newMetricsLock := sync.Mutex{}

	// Fetch what's missing.
	wgInstances := sync.WaitGroup{}
	for _, project := range toFetch {
		newMetrics[project] = metrics.NewMetricSet(nil)

		// Get the instances.
		instances, err := instanceLoadNodeProjectAll(d.State(), project, instancetype.Any)
		if err != nil {
			return response.SmartError(err)
		}

		for _, inst := range instances {
			// Ignore stopped instances.
			if !inst.IsRunning() {
				continue
			}

			wgInstances.Add(1)
			go func(inst instance.Instance) {
				defer wgInstances.Done()

				instanceMetrics, err := inst.Metrics()
				if err != nil {
					logger.Warn("Failed to get instance metrics", log.Ctx{"instance": inst.Name(), "project": inst.Project(), "err": err})
					return
				}

				// Add the metrics.
				newMetricsLock.Lock()
				defer newMetricsLock.Unlock()

				newMetrics[inst.Project()].Merge(instanceMetrics)
			}(inst)
		}
	}

	wgInstances.Wait()

	// Put the new data in the global cache and in response.
	metricsCacheLock.Lock()

	if metricsCache == nil {
		metricsCache = map[string]metricsCacheEntry{}
	}

	for project, entries := range newMetrics {
		metricsCache[project] = metricsCacheEntry{
			expiry:  time.Now().Add(8 * time.Second),
			metrics: entries,
		}

		metricSet.Merge(entries)
	}
	metricsCacheLock.Unlock()

	return response.SyncResponsePlain(true, metricSet.String())
}
<|endoftext|>