<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubelet

import (
	"fmt"
	"strconv"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
	kubecontainer "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/container"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/probe"
	execprobe "github.com/GoogleCloudPlatform/kubernetes/pkg/probe/exec"
	httprobe "github.com/GoogleCloudPlatform/kubernetes/pkg/probe/http"
	tcprobe "github.com/GoogleCloudPlatform/kubernetes/pkg/probe/tcp"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/exec"

	"github.com/golang/glog"
)

const maxProbeRetries = 3

// prober helps to check the liveness/readiness of a container.
type prober struct {
	exec   execprobe.ExecProber
	http   httprobe.HTTPProber
	tcp    tcprobe.TCPProber
	runner dockertools.ContainerCommandRunner

	readinessManager *kubecontainer.ReadinessManager
	refManager       *kubecontainer.RefManager
	recorder         record.EventRecorder
}

// NewProber creates a Prober, it takes a command runner and
// several container info managers.
func newProber(
	runner dockertools.ContainerCommandRunner,
	readinessManager *kubecontainer.ReadinessManager,
	refManager *kubecontainer.RefManager,
	recorder record.EventRecorder) kubecontainer.Prober {

	return &prober{
		exec:   execprobe.New(),
		http:   httprobe.New(),
		tcp:    tcprobe.New(),
		runner: runner,

		readinessManager: readinessManager,
		refManager:       refManager,
		recorder:         recorder,
	}
}

// Probe checks the liveness/readiness of the given container.
// If the container's liveness probe is unsuccessful, set readiness to false.
// If liveness is successful, do a readiness check and set readiness accordingly.
func (pb *prober) Probe(pod *api.Pod, status api.PodStatus, container api.Container, containerID string, createdAt int64) (probe.Result, error) {
	// Probe liveness.
	live, err := pb.probeLiveness(pod, status, container, containerID, createdAt)
	if err != nil {
		glog.V(1).Infof("Liveness probe errored: %v", err)
		pb.readinessManager.SetReadiness(containerID, false)
		return probe.Unknown, err
	}
	if live != probe.Success {
		glog.V(1).Infof("Liveness probe unsuccessful: %v", live)
		pb.readinessManager.SetReadiness(containerID, false)
		return live, nil
	}

	// Probe readiness.
	ready, err := pb.probeReadiness(pod, status, container, containerID, createdAt)
	if err == nil && ready == probe.Success {
		glog.V(3).Infof("Readiness probe successful: %v", ready)
		pb.readinessManager.SetReadiness(containerID, true)
		return probe.Success, nil
	}

	glog.V(1).Infof("Readiness probe failed/errored: %v, %v", ready, err)
	pb.readinessManager.SetReadiness(containerID, false)

	ref, ok := pb.refManager.GetRef(containerID)
	if !ok {
		glog.Warningf("No ref for pod '%v' - '%v'", containerID, container.Name)
		return probe.Success, err
	}

	if ready != probe.Success {
		pb.recorder.Eventf(ref, "unhealthy", "Readiness Probe Failed %v - %v", containerID, container.Name)
	}

	return probe.Success, nil
}

// probeLiveness probes the liveness of a container.
// If the initalDelay since container creation on liveness probe has not passed the probe will return probe.Success.
func (pb *prober) probeLiveness(pod *api.Pod, status api.PodStatus, container api.Container, containerID string, createdAt int64) (probe.Result, error) {
	p := container.LivenessProbe
	if p == nil {
		return probe.Success, nil
	}
	if time.Now().Unix()-createdAt < p.InitialDelaySeconds {
		return probe.Success, nil
	}
	return pb.runProbeWithRetries(p, pod, status, container, containerID, maxProbeRetries)
}

// probeReadiness probes the readiness of a container.
// If the initial delay on the readiness probe has not passed the probe will return probe.Failure.
func (pb *prober) probeReadiness(pod *api.Pod, status api.PodStatus, container api.Container, containerID string, createdAt int64) (probe.Result, error) {
	p := container.ReadinessProbe
	if p == nil {
		return probe.Success, nil
	}
	if time.Now().Unix()-createdAt < p.InitialDelaySeconds {
		return probe.Failure, nil
	}
	return pb.runProbeWithRetries(p, pod, status, container, containerID, maxProbeRetries)
}

// runProbeWithRetries tries to probe the container in a finite loop, it returns the last result
// if it never succeeds.
func (pb *prober) runProbeWithRetries(p *api.Probe, pod *api.Pod, status api.PodStatus, container api.Container, containerID string, retires int) (probe.Result, error) {
	var err error
	var result probe.Result
	for i := 0; i < retires; i++ {
		result, err = pb.runProbe(p, pod, status, container, containerID)
		if result == probe.Success {
			return probe.Success, nil
		}
	}
	return result, err
}

func (pb *prober) runProbe(p *api.Probe, pod *api.Pod, status api.PodStatus, container api.Container, containerID string) (probe.Result, error) {
	timeout := time.Duration(p.TimeoutSeconds) * time.Second
	if p.Exec != nil {
		glog.V(4).Infof("Exec-Probe Pod: %v, Container: %v", pod, container)
		return pb.exec.Probe(pb.newExecInContainer(pod, container, containerID))
	}
	if p.HTTPGet != nil {
		port, err := extractPort(p.HTTPGet.Port, container)
		if err != nil {
			return probe.Unknown, err
		}
		host, port, path := extractGetParams(p.HTTPGet, status, port)
		glog.V(4).Infof("HTTP-Probe Host: %v, Port: %v, Path: %v", host, port, path)
		return pb.http.Probe(host, port, path, timeout)
	}
	if p.TCPSocket != nil {
		port, err := extractPort(p.TCPSocket.Port, container)
		if err != nil {
			return probe.Unknown, err
		}
		glog.V(4).Infof("TCP-Probe PodIP: %v, Port: %v, Timeout: %v", status.PodIP, port, timeout)
		return pb.tcp.Probe(status.PodIP, port, timeout)
	}
	glog.Warningf("Failed to find probe builder for %s %+v", container.Name, container.LivenessProbe)
	return probe.Unknown, nil
}

func extractGetParams(action *api.HTTPGetAction, status api.PodStatus, port int) (string, int, string) {
	host := action.Host
	if host == "" {
		host = status.PodIP
	}
	return host, port, action.Path
}

func extractPort(param util.IntOrString, container api.Container) (int, error) {
	port := -1
	var err error
	switch param.Kind {
	case util.IntstrInt:
		port := param.IntVal
		if port > 0 && port < 65536 {
			return port, nil
		}
		return port, fmt.Errorf("invalid port number: %v", port)
	case util.IntstrString:
		port = findPortByName(container, param.StrVal)
		if port == -1 {
			// Last ditch effort - maybe it was an int stored as string?
			if port, err = strconv.Atoi(param.StrVal); err != nil {
				return port, err
			}
		}
		if port > 0 && port < 65536 {
			return port, nil
		}
		return port, fmt.Errorf("invalid port number: %v", port)
	default:
		return port, fmt.Errorf("IntOrString had no kind: %+v", param)
	}
}

// findPortByName is a helper function to look up a port in a container by name.
// Returns the HostPort if found, -1 if not found.
func findPortByName(container api.Container, portName string) int {
	for _, port := range container.Ports {
		if port.Name == portName {
			return port.HostPort
		}
	}
	return -1
}

type execInContainer struct {
	run func() ([]byte, error)
}

func (p *prober) newExecInContainer(pod *api.Pod, container api.Container, containerID string) exec.Cmd {
	return execInContainer{func() ([]byte, error) {
		return p.runner.RunInContainer(containerID, container.LivenessProbe.Exec.Command)
	}}
}

func (eic execInContainer) CombinedOutput() ([]byte, error) {
	return eic.run()
}

func (eic execInContainer) SetDir(dir string) {
	//unimplemented
}
<commit_msg>Fixes issue #7352<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubelet

import (
	"fmt"
	"strconv"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
	kubecontainer "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/container"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/probe"
	execprobe "github.com/GoogleCloudPlatform/kubernetes/pkg/probe/exec"
	httprobe "github.com/GoogleCloudPlatform/kubernetes/pkg/probe/http"
	tcprobe "github.com/GoogleCloudPlatform/kubernetes/pkg/probe/tcp"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/exec"

	"github.com/golang/glog"
)

const maxProbeRetries = 3

// prober helps to check the liveness/readiness of a container.
type prober struct {
	exec   execprobe.ExecProber
	http   httprobe.HTTPProber
	tcp    tcprobe.TCPProber
	runner dockertools.ContainerCommandRunner

	readinessManager *kubecontainer.ReadinessManager
	refManager       *kubecontainer.RefManager
	recorder         record.EventRecorder
}

// NewProber creates a Prober, it takes a command runner and
// several container info managers.
func newProber(
	runner dockertools.ContainerCommandRunner,
	readinessManager *kubecontainer.ReadinessManager,
	refManager *kubecontainer.RefManager,
	recorder record.EventRecorder) kubecontainer.Prober {

	return &prober{
		exec:   execprobe.New(),
		http:   httprobe.New(),
		tcp:    tcprobe.New(),
		runner: runner,

		readinessManager: readinessManager,
		refManager:       refManager,
		recorder:         recorder,
	}
}

// Probe checks the liveness/readiness of the given container.
// If the container's liveness probe is unsuccessful, set readiness to false.
// If liveness is successful, do a readiness check and set readiness accordingly.
func (pb *prober) Probe(pod *api.Pod, status api.PodStatus, container api.Container, containerID string, createdAt int64) (probe.Result, error) {
	// Probe liveness.
	live, err := pb.probeLiveness(pod, status, container, containerID, createdAt)
	if err != nil {
		glog.V(1).Infof("Liveness probe errored: %v", err)
		pb.readinessManager.SetReadiness(containerID, false)
		return probe.Unknown, err
	}
	if live != probe.Success {
		glog.V(1).Infof("Liveness probe unsuccessful: %v", live)
		pb.readinessManager.SetReadiness(containerID, false)
		return live, nil
	}

	// Probe readiness.
	ready, err := pb.probeReadiness(pod, status, container, containerID, createdAt)
	if err == nil && ready == probe.Success {
		glog.V(3).Infof("Readiness probe successful: %v", ready)
		pb.readinessManager.SetReadiness(containerID, true)
		return probe.Success, nil
	}

	glog.V(1).Infof("Readiness probe failed/errored: %v, %v", ready, err)
	pb.readinessManager.SetReadiness(containerID, false)

	ref, ok := pb.refManager.GetRef(containerID)
	if !ok {
		glog.Warningf("No ref for pod '%v' - '%v'", containerID, container.Name)
		return probe.Success, err
	}

	if ready != probe.Success {
		pb.recorder.Eventf(ref, "unhealthy", "Readiness Probe Failed %v - %v", containerID, container.Name)
	}

	return probe.Success, nil
}

// probeLiveness probes the liveness of a container.
// If the initalDelay since container creation on liveness probe has not passed the probe will return probe.Success.
func (pb *prober) probeLiveness(pod *api.Pod, status api.PodStatus, container api.Container, containerID string, createdAt int64) (probe.Result, error) {
	p := container.LivenessProbe
	if p == nil {
		return probe.Success, nil
	}
	if time.Now().Unix()-createdAt < p.InitialDelaySeconds {
		return probe.Success, nil
	}
	return pb.runProbeWithRetries(p, pod, status, container, containerID, maxProbeRetries)
}

// probeReadiness probes the readiness of a container.
// If the initial delay on the readiness probe has not passed the probe will return probe.Failure.
func (pb *prober) probeReadiness(pod *api.Pod, status api.PodStatus, container api.Container, containerID string, createdAt int64) (probe.Result, error) {
	p := container.ReadinessProbe
	if p == nil {
		return probe.Success, nil
	}
	if time.Now().Unix()-createdAt < p.InitialDelaySeconds {
		return probe.Failure, nil
	}
	return pb.runProbeWithRetries(p, pod, status, container, containerID, maxProbeRetries)
}

// runProbeWithRetries tries to probe the container in a finite loop, it returns the last result
// if it never succeeds.
func (pb *prober) runProbeWithRetries(p *api.Probe, pod *api.Pod, status api.PodStatus, container api.Container, containerID string, retires int) (probe.Result, error) {
	var err error
	var result probe.Result
	for i := 0; i < retires; i++ {
		result, err = pb.runProbe(p, pod, status, container, containerID)
		if result == probe.Success {
			return probe.Success, nil
		}
	}
	return result, err
}

func (pb *prober) runProbe(p *api.Probe, pod *api.Pod, status api.PodStatus, container api.Container, containerID string) (probe.Result, error) {
	timeout := time.Duration(p.TimeoutSeconds) * time.Second
	if p.Exec != nil {
		glog.V(4).Infof("Exec-Probe Pod: %v, Container: %v", pod, container)
		return pb.exec.Probe(pb.newExecInContainer(pod, container, containerID, p.Exec.Command))
	}
	if p.HTTPGet != nil {
		port, err := extractPort(p.HTTPGet.Port, container)
		if err != nil {
			return probe.Unknown, err
		}
		host, port, path := extractGetParams(p.HTTPGet, status, port)
		glog.V(4).Infof("HTTP-Probe Host: %v, Port: %v, Path: %v", host, port, path)
		return pb.http.Probe(host, port, path, timeout)
	}
	if p.TCPSocket != nil {
		port, err := extractPort(p.TCPSocket.Port, container)
		if err != nil {
			return probe.Unknown, err
		}
		glog.V(4).Infof("TCP-Probe PodIP: %v, Port: %v, Timeout: %v", status.PodIP, port, timeout)
		return pb.tcp.Probe(status.PodIP, port, timeout)
	}
	glog.Warningf("Failed to find probe builder for %s %+v", container.Name, container.LivenessProbe)
	return probe.Unknown, nil
}

func extractGetParams(action *api.HTTPGetAction, status api.PodStatus, port int) (string, int, string) {
	host := action.Host
	if host == "" {
		host = status.PodIP
	}
	return host, port, action.Path
}

func extractPort(param util.IntOrString, container api.Container) (int, error) {
	port := -1
	var err error
	switch param.Kind {
	case util.IntstrInt:
		port := param.IntVal
		if port > 0 && port < 65536 {
			return port, nil
		}
		return port, fmt.Errorf("invalid port number: %v", port)
	case util.IntstrString:
		port = findPortByName(container, param.StrVal)
		if port == -1 {
			// Last ditch effort - maybe it was an int stored as string?
			if port, err = strconv.Atoi(param.StrVal); err != nil {
				return port, err
			}
		}
		if port > 0 && port < 65536 {
			return port, nil
		}
		return port, fmt.Errorf("invalid port number: %v", port)
	default:
		return port, fmt.Errorf("IntOrString had no kind: %+v", param)
	}
}

// findPortByName is a helper function to look up a port in a container by name.
// Returns the HostPort if found, -1 if not found.
func findPortByName(container api.Container, portName string) int {
	for _, port := range container.Ports {
		if port.Name == portName {
			return port.HostPort
		}
	}
	return -1
}

type execInContainer struct {
	run func() ([]byte, error)
}

func (p *prober) newExecInContainer(pod *api.Pod, container api.Container, containerID string, cmd []string) exec.Cmd {
	return execInContainer{func() ([]byte, error) {
		return p.runner.RunInContainer(containerID, cmd)
	}}
}

func (eic execInContainer) CombinedOutput() ([]byte, error) {
	return eic.run()
}

func (eic execInContainer) SetDir(dir string) {
	//unimplemented
}
<|endoftext|>


<commit_before>package loader

import (
	"github.com/hellofresh/janus/pkg/api"
	"github.com/hellofresh/janus/pkg/oauth"
	"github.com/hellofresh/janus/pkg/plugin"
	"github.com/hellofresh/janus/pkg/proxy"
	"github.com/hellofresh/janus/pkg/router"
	"github.com/hellofresh/janus/pkg/store"
)

// Params initialization options.
type Params struct {
	Router      router.Router
	Storage     store.Store
	APIRepo     api.Repository
	OAuthRepo   oauth.Repository
	ProxyParams proxy.Params
}

func Load(params Params) {
	pluginLoader := plugin.NewLoader()
	pluginLoader.Add(
		plugin.NewRateLimit(params.Storage),
		plugin.NewCORS(),
		plugin.NewOAuth2(params.OAuthRepo, params.Storage),
		plugin.NewCompression(),
	)

	prx := proxy.WithParams(params.ProxyParams)

	// create proxy register
	register := proxy.NewRegister(params.Router, prx)

	apiLoader := NewAPILoader(register, pluginLoader)
	apiLoader.LoadDefinitions(params.APIRepo)

	oauthLoader := NewOAuthLoader(register, params.Storage)
	oauthLoader.LoadDefinitions(params.OAuthRepo)
}
<commit_msg>Added missing comment<commit_after>package loader

import (
	"github.com/hellofresh/janus/pkg/api"
	"github.com/hellofresh/janus/pkg/oauth"
	"github.com/hellofresh/janus/pkg/plugin"
	"github.com/hellofresh/janus/pkg/proxy"
	"github.com/hellofresh/janus/pkg/router"
	"github.com/hellofresh/janus/pkg/store"
)

// Params initialization options.
type Params struct {
	Router      router.Router
	Storage     store.Store
	APIRepo     api.Repository
	OAuthRepo   oauth.Repository
	ProxyParams proxy.Params
}

// Load loads all the basic components and definitions into a router
func Load(params Params) {
	pluginLoader := plugin.NewLoader()
	pluginLoader.Add(
		plugin.NewRateLimit(params.Storage),
		plugin.NewCORS(),
		plugin.NewOAuth2(params.OAuthRepo, params.Storage),
		plugin.NewCompression(),
	)

	prx := proxy.WithParams(params.ProxyParams)

	// create proxy register
	register := proxy.NewRegister(params.Router, prx)

	apiLoader := NewAPILoader(register, pluginLoader)
	apiLoader.LoadDefinitions(params.APIRepo)

	oauthLoader := NewOAuthLoader(register, params.Storage)
	oauthLoader.LoadDefinitions(params.OAuthRepo)
}
<|endoftext|>


<commit_before>package loader

import (
	"github.com/hellofresh/janus/pkg/api"
	"github.com/hellofresh/janus/pkg/oauth"
	"github.com/hellofresh/janus/pkg/plugin"
	"github.com/hellofresh/janus/pkg/proxy"
	"github.com/hellofresh/janus/pkg/router"
	"github.com/hellofresh/janus/pkg/store"
	"github.com/hellofresh/janus/pkg/web"
	stats "github.com/hellofresh/stats-go"

	// this is needed to call the init function on each plugin
	_ "github.com/hellofresh/janus/pkg/plugin/compression"
	_ "github.com/hellofresh/janus/pkg/plugin/cors"
	_ "github.com/hellofresh/janus/pkg/plugin/oauth2"
	_ "github.com/hellofresh/janus/pkg/plugin/rate"
	_ "github.com/hellofresh/janus/pkg/plugin/requesttransformer"
)

// Params initialization options.
type Params struct {
	Router      router.Router
	Storage     store.Store
	APIRepo     api.Repository
	OAuthRepo   oauth.Repository
	StatsClient stats.Client
	ProxyParams proxy.Params
}

// Load loads all the basic components and definitions into a router
func Load(params Params) {
	// create proxy register
	register := proxy.NewRegister(params.Router, params.ProxyParams)

	apiLoader := NewAPILoader(register, plugin.Params{
		Router:    params.Router,
		Storage:   params.Storage,
		APIRepo:   params.APIRepo,
		OAuthRepo: params.OAuthRepo,
	})
	apiLoader.LoadDefinitions(params.APIRepo)

	oauthLoader := NewOAuthLoader(register, params.Storage)
	oauthLoader.LoadDefinitions(params.OAuthRepo)

	// some routers may panic when have empty routes list, so add one dummy 404 route to avoid this
	if params.Router.RoutesCount() < 1 {
		params.Router.Any("/", web.NotFound)
	}
}
<commit_msg>Added missing stats client<commit_after>package loader

import (
	"github.com/hellofresh/janus/pkg/api"
	"github.com/hellofresh/janus/pkg/oauth"
	"github.com/hellofresh/janus/pkg/plugin"
	"github.com/hellofresh/janus/pkg/proxy"
	"github.com/hellofresh/janus/pkg/router"
	"github.com/hellofresh/janus/pkg/store"
	"github.com/hellofresh/janus/pkg/web"
	stats "github.com/hellofresh/stats-go"

	// this is needed to call the init function on each plugin
	_ "github.com/hellofresh/janus/pkg/plugin/compression"
	_ "github.com/hellofresh/janus/pkg/plugin/cors"
	_ "github.com/hellofresh/janus/pkg/plugin/oauth2"
	_ "github.com/hellofresh/janus/pkg/plugin/rate"
	_ "github.com/hellofresh/janus/pkg/plugin/requesttransformer"
)

// Params initialization options.
type Params struct {
	Router      router.Router
	Storage     store.Store
	APIRepo     api.Repository
	OAuthRepo   oauth.Repository
	StatsClient stats.Client
	ProxyParams proxy.Params
}

// Load loads all the basic components and definitions into a router
func Load(params Params) {
	// create proxy register
	register := proxy.NewRegister(params.Router, params.ProxyParams)

	apiLoader := NewAPILoader(register, plugin.Params{
		Router:      params.Router,
		Storage:     params.Storage,
		APIRepo:     params.APIRepo,
		OAuthRepo:   params.OAuthRepo,
		StatsClient: params.StatsClient,
	})
	apiLoader.LoadDefinitions(params.APIRepo)

	oauthLoader := NewOAuthLoader(register, params.Storage)
	oauthLoader.LoadDefinitions(params.OAuthRepo)

	// some routers may panic when have empty routes list, so add one dummy 404 route to avoid this
	if params.Router.RoutesCount() < 1 {
		params.Router.Any("/", web.NotFound)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mapper

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"sync"
	"time"

	"github.com/go-kit/log"
	"github.com/prometheus/client_golang/prometheus"
	"gopkg.in/yaml.v2"

	"github.com/prometheus/statsd_exporter/pkg/level"
	"github.com/prometheus/statsd_exporter/pkg/mapper/fsm"
)

var (
	// The first segment of a match cannot start with a number
	statsdMetricRE = `[a-zA-Z_]([a-zA-Z0-9_\-])*`
	// The subsequent segments of a match can start with a number
	// See https://github.com/prometheus/statsd_exporter/issues/328
	statsdMetricSubsequentRE = `[a-zA-Z0-9_]([a-zA-Z0-9_\-])*`
	templateReplaceRE        = `(\$\{?\d+\}?)`

	metricLineRE = regexp.MustCompile(`^(\*|` + statsdMetricRE + `)(\.\*|\.` + statsdMetricSubsequentRE + `)*$`)
	metricNameRE = regexp.MustCompile(`^([a-zA-Z_]|` + templateReplaceRE + `)([a-zA-Z0-9_]|` + templateReplaceRE + `)*$`)
	labelNameRE  = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]+$`)
)

type MetricMapper struct {
	Registerer prometheus.Registerer
	Defaults   mapperConfigDefaults `yaml:"defaults"`
	Mappings   []MetricMapping      `yaml:"mappings"`
	FSM        *fsm.FSM
	doFSM      bool
	doRegex    bool
	cache      MetricMapperCache
	mutex      sync.RWMutex

	MappingsCount prometheus.Gauge

	Logger log.Logger
}

type SummaryOptions struct {
	Quantiles  []metricObjective `yaml:"quantiles"`
	MaxAge     time.Duration     `yaml:"max_age"`
	AgeBuckets uint32            `yaml:"age_buckets"`
	BufCap     uint32            `yaml:"buf_cap"`
}

type HistogramOptions struct {
	Buckets []float64 `yaml:"buckets"`
}

type metricObjective struct {
	Quantile float64 `yaml:"quantile"`
	Error    float64 `yaml:"error"`
}

var defaultQuantiles = []metricObjective{
	{Quantile: 0.5, Error: 0.05},
	{Quantile: 0.9, Error: 0.01},
	{Quantile: 0.99, Error: 0.001},
}

func (m *MetricMapper) InitFromYAMLString(fileContents string) error {
	var n MetricMapper

	if err := yaml.Unmarshal([]byte(fileContents), &n); err != nil {
		return err
	}

	if len(n.Defaults.HistogramOptions.Buckets) == 0 {
		n.Defaults.HistogramOptions.Buckets = prometheus.DefBuckets
	}

	if len(n.Defaults.SummaryOptions.Quantiles) == 0 {
		n.Defaults.SummaryOptions.Quantiles = defaultQuantiles
	}

	if n.Defaults.MatchType == MatchTypeDefault {
		n.Defaults.MatchType = MatchTypeGlob
	}

	remainingMappingsCount := len(n.Mappings)

	n.FSM = fsm.NewFSM([]string{string(MetricTypeCounter), string(MetricTypeGauge), string(MetricTypeObserver)},
		remainingMappingsCount, n.Defaults.GlobDisableOrdering)

	for i := range n.Mappings {
		remainingMappingsCount--

		currentMapping := &n.Mappings[i]

		// check that label is correct
		for k := range currentMapping.Labels {
			if !labelNameRE.MatchString(k) {
				return fmt.Errorf("invalid label key: %s", k)
			}
		}

		if currentMapping.Name == "" {
			return fmt.Errorf("line %d: metric mapping didn't set a metric name", i)
		}

		if !metricNameRE.MatchString(currentMapping.Name) {
			return fmt.Errorf("metric name '%s' doesn't match regex '%s'", currentMapping.Name, metricNameRE)
		}

		if currentMapping.MatchType == "" {
			currentMapping.MatchType = n.Defaults.MatchType
		}

		if currentMapping.Action == "" {
			currentMapping.Action = ActionTypeMap
		}

		if currentMapping.MatchType == MatchTypeGlob {
			n.doFSM = true
			if !metricLineRE.MatchString(currentMapping.Match) {
				return fmt.Errorf("invalid match: %s", currentMapping.Match)
			}

			captureCount := n.FSM.AddState(currentMapping.Match, string(currentMapping.MatchMetricType),
				remainingMappingsCount, currentMapping)

			currentMapping.nameFormatter = fsm.NewTemplateFormatter(currentMapping.Name, captureCount)

			labelKeys := make([]string, len(currentMapping.Labels))
			labelFormatters := make([]*fsm.TemplateFormatter, len(currentMapping.Labels))
			labelIndex := 0
			for label, valueExpr := range currentMapping.Labels {
				labelKeys[labelIndex] = label
				labelFormatters[labelIndex] = fsm.NewTemplateFormatter(valueExpr, captureCount)
				labelIndex++
			}
			currentMapping.labelFormatters = labelFormatters
			currentMapping.labelKeys = labelKeys
		} else {
			if regex, err := regexp.Compile(currentMapping.Match); err != nil {
				return fmt.Errorf("invalid regex %s in mapping: %v", currentMapping.Match, err)
			} else {
				currentMapping.regex = regex
			}
			n.doRegex = true
		}

		if currentMapping.ObserverType == "" {
			currentMapping.ObserverType = n.Defaults.ObserverType
		}

		if currentMapping.LegacyQuantiles != nil &&
			(currentMapping.SummaryOptions == nil || currentMapping.SummaryOptions.Quantiles != nil) {
			level.Warn(m.Logger).Log("msg", "using the top level quantiles is deprecated.  Please use quantiles in the summary_options hierarchy")
		}

		if currentMapping.LegacyBuckets != nil &&
			(currentMapping.HistogramOptions == nil || currentMapping.HistogramOptions.Buckets != nil) {
			level.Warn(m.Logger).Log("msg", "using the top level buckets is deprecated.  Please use buckets in the histogram_options hierarchy")
		}

		if currentMapping.SummaryOptions != nil &&
			currentMapping.LegacyQuantiles != nil &&
			currentMapping.SummaryOptions.Quantiles != nil {
			return fmt.Errorf("cannot use quantiles in both the top level and summary options at the same time in %s", currentMapping.Match)
		}

		if currentMapping.HistogramOptions != nil &&
			currentMapping.LegacyBuckets != nil &&
			currentMapping.HistogramOptions.Buckets != nil {
			return fmt.Errorf("cannot use buckets in both the top level and histogram options at the same time in %s", currentMapping.Match)
		}

		if currentMapping.ObserverType == ObserverTypeHistogram {
			if currentMapping.SummaryOptions != nil {
				return fmt.Errorf("cannot use histogram observer and summary options at the same time")
			}
			if currentMapping.HistogramOptions == nil {
				currentMapping.HistogramOptions = &HistogramOptions{}
			}
			if currentMapping.LegacyBuckets != nil && len(currentMapping.LegacyBuckets) != 0 {
				currentMapping.HistogramOptions.Buckets = currentMapping.LegacyBuckets
			}
			if currentMapping.HistogramOptions.Buckets == nil || len(currentMapping.HistogramOptions.Buckets) == 0 {
				currentMapping.HistogramOptions.Buckets = n.Defaults.HistogramOptions.Buckets
			}
		}

		if currentMapping.ObserverType == ObserverTypeSummary {
			if currentMapping.HistogramOptions != nil {
				return fmt.Errorf("cannot use summary observer and histogram options at the same time")
			}
			if currentMapping.SummaryOptions == nil {
				currentMapping.SummaryOptions = &SummaryOptions{}
			}
			if currentMapping.LegacyQuantiles != nil && len(currentMapping.LegacyQuantiles) != 0 {
				currentMapping.SummaryOptions.Quantiles = currentMapping.LegacyQuantiles
			}
			if currentMapping.SummaryOptions.Quantiles == nil || len(currentMapping.SummaryOptions.Quantiles) == 0 {
				currentMapping.SummaryOptions.Quantiles = n.Defaults.SummaryOptions.Quantiles
			}
			if currentMapping.SummaryOptions.MaxAge == 0 {
				currentMapping.SummaryOptions.MaxAge = n.Defaults.SummaryOptions.MaxAge
			}
			if currentMapping.SummaryOptions.AgeBuckets == 0 {
				currentMapping.SummaryOptions.AgeBuckets = n.Defaults.SummaryOptions.AgeBuckets
			}
			if currentMapping.SummaryOptions.BufCap == 0 {
				currentMapping.SummaryOptions.BufCap = n.Defaults.SummaryOptions.BufCap
			}
		}

		if currentMapping.Ttl == 0 && n.Defaults.Ttl > 0 {
			currentMapping.Ttl = n.Defaults.Ttl
		}
	}

	m.mutex.Lock()
	defer m.mutex.Unlock()

	m.Defaults = n.Defaults
	m.Mappings = n.Mappings

	// Reset the cache since this function can be used to reload config
	if m.cache != nil {
		m.cache.Reset()
	}

	if n.doFSM {
		var mappings []string
		for _, mapping := range n.Mappings {
			if mapping.MatchType == MatchTypeGlob {
				mappings = append(mappings, mapping.Match)
			}
		}
		n.FSM.BacktrackingNeeded = fsm.TestIfNeedBacktracking(mappings, n.FSM.OrderingDisabled, m.Logger)

		m.FSM = n.FSM
		m.doRegex = n.doRegex
	}
	m.doFSM = n.doFSM

	if m.MappingsCount != nil {
		m.MappingsCount.Set(float64(len(n.Mappings)))
	}

	if m.Logger == nil {
		m.Logger = log.NewNopLogger()
	}

	return nil
}

func (m *MetricMapper) InitFromFile(fileName string) error {
	mappingStr, err := ioutil.ReadFile(fileName)
	if err != nil {
		return err
	}

	return m.InitFromYAMLString(string(mappingStr))
}

// UseCache tells the mapper to use a cache that implements the MetricMapperCache interface.
// This cache MUST be thread-safe!
func (m *MetricMapper) UseCache(cache MetricMapperCache) {
	m.mutex.Lock()
	defer m.mutex.Unlock()
	m.cache = cache
}

func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricType) (*MetricMapping, prometheus.Labels, bool) {
	m.mutex.RLock()
	defer m.mutex.RUnlock()

	// only use a cache if one is present
	if m.cache != nil {
		result, cached := m.cache.Get(formatKey(statsdMetric, statsdMetricType))
		if cached {
			r := result.(MetricMapperCacheResult)
			return r.Mapping, r.Labels, r.Matched
		}
	}

	// glob matching
	if m.doFSM {
		finalState, captures := m.FSM.GetMapping(statsdMetric, string(statsdMetricType))
		if finalState != nil && finalState.Result != nil {
			v := finalState.Result.(*MetricMapping)
			result := copyMetricMapping(v)
			result.Name = result.nameFormatter.Format(captures)

			labels := prometheus.Labels{}
			for index, formatter := range result.labelFormatters {
				labels[result.labelKeys[index]] = formatter.Format(captures)
			}

			r := MetricMapperCacheResult{
				Mapping: result,
				Matched: true,
				Labels:  labels,
			}
			// add match to cache
			if m.cache != nil {
				m.cache.Add(formatKey(statsdMetric, statsdMetricType), r)
			}

			return result, labels, true
		} else if !m.doRegex {
			// if there's no regex match type, return immediately
			// Add miss to cache
			if m.cache != nil {
				m.cache.Add(formatKey(statsdMetric, statsdMetricType), MetricMapperCacheResult{})
			}
			return nil, nil, false
		}
	}

	// regex matching
	for _, mapping := range m.Mappings {
		// if a rule don't have regex matching type, the regex field is unset
		if mapping.regex == nil {
			continue
		}
		matches := mapping.regex.FindStringSubmatchIndex(statsdMetric)
		if len(matches) == 0 {
			continue
		}

		mapping.Name = string(mapping.regex.ExpandString(
			[]byte{},
			mapping.Name,
			statsdMetric,
			matches,
		))

		if mt := mapping.MatchMetricType; mt != "" && mt != statsdMetricType {
			continue
		}

		labels := prometheus.Labels{}
		for label, valueExpr := range mapping.Labels {
			value := mapping.regex.ExpandString([]byte{}, valueExpr, statsdMetric, matches)
			labels[label] = string(value)
		}

		r := MetricMapperCacheResult{
			Mapping: &mapping,
			Matched: true,
			Labels:  labels,
		}
		// Add Match to cache
		if m.cache != nil {
			m.cache.Add(formatKey(statsdMetric, statsdMetricType), r)
		}

		return &mapping, labels, true
	}

	// Add Miss to cache
	if m.cache != nil {
		m.cache.Add(formatKey(statsdMetric, statsdMetricType), MetricMapperCacheResult{})
	}
	return nil, nil, false
}

// make a shallow copy so that we do not overwrite name
// as multiple names can be matched by same mapping
func copyMetricMapping(in *MetricMapping) *MetricMapping {
	out := *in
	return &out
}
<commit_msg>mapper: Make sure we have a logger before backtracking check<commit_after>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mapper

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"sync"
	"time"

	"github.com/go-kit/log"
	"github.com/prometheus/client_golang/prometheus"
	"gopkg.in/yaml.v2"

	"github.com/prometheus/statsd_exporter/pkg/level"
	"github.com/prometheus/statsd_exporter/pkg/mapper/fsm"
)

var (
	// The first segment of a match cannot start with a number
	statsdMetricRE = `[a-zA-Z_]([a-zA-Z0-9_\-])*`
	// The subsequent segments of a match can start with a number
	// See https://github.com/prometheus/statsd_exporter/issues/328
	statsdMetricSubsequentRE = `[a-zA-Z0-9_]([a-zA-Z0-9_\-])*`
	templateReplaceRE        = `(\$\{?\d+\}?)`

	metricLineRE = regexp.MustCompile(`^(\*|` + statsdMetricRE + `)(\.\*|\.` + statsdMetricSubsequentRE + `)*$`)
	metricNameRE = regexp.MustCompile(`^([a-zA-Z_]|` + templateReplaceRE + `)([a-zA-Z0-9_]|` + templateReplaceRE + `)*$`)
	labelNameRE  = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]+$`)
)

type MetricMapper struct {
	Registerer prometheus.Registerer
	Defaults   mapperConfigDefaults `yaml:"defaults"`
	Mappings   []MetricMapping      `yaml:"mappings"`
	FSM        *fsm.FSM
	doFSM      bool
	doRegex    bool
	cache      MetricMapperCache
	mutex      sync.RWMutex

	MappingsCount prometheus.Gauge

	Logger log.Logger
}

type SummaryOptions struct {
	Quantiles  []metricObjective `yaml:"quantiles"`
	MaxAge     time.Duration     `yaml:"max_age"`
	AgeBuckets uint32            `yaml:"age_buckets"`
	BufCap     uint32            `yaml:"buf_cap"`
}

type HistogramOptions struct {
	Buckets []float64 `yaml:"buckets"`
}

type metricObjective struct {
	Quantile float64 `yaml:"quantile"`
	Error    float64 `yaml:"error"`
}

var defaultQuantiles = []metricObjective{
	{Quantile: 0.5, Error: 0.05},
	{Quantile: 0.9, Error: 0.01},
	{Quantile: 0.99, Error: 0.001},
}

func (m *MetricMapper) InitFromYAMLString(fileContents string) error {
	var n MetricMapper

	if err := yaml.Unmarshal([]byte(fileContents), &n); err != nil {
		return err
	}

	if len(n.Defaults.HistogramOptions.Buckets) == 0 {
		n.Defaults.HistogramOptions.Buckets = prometheus.DefBuckets
	}

	if len(n.Defaults.SummaryOptions.Quantiles) == 0 {
		n.Defaults.SummaryOptions.Quantiles = defaultQuantiles
	}

	if n.Defaults.MatchType == MatchTypeDefault {
		n.Defaults.MatchType = MatchTypeGlob
	}

	remainingMappingsCount := len(n.Mappings)

	n.FSM = fsm.NewFSM([]string{string(MetricTypeCounter), string(MetricTypeGauge), string(MetricTypeObserver)},
		remainingMappingsCount, n.Defaults.GlobDisableOrdering)

	for i := range n.Mappings {
		remainingMappingsCount--

		currentMapping := &n.Mappings[i]

		// check that label is correct
		for k := range currentMapping.Labels {
			if !labelNameRE.MatchString(k) {
				return fmt.Errorf("invalid label key: %s", k)
			}
		}

		if currentMapping.Name == "" {
			return fmt.Errorf("line %d: metric mapping didn't set a metric name", i)
		}

		if !metricNameRE.MatchString(currentMapping.Name) {
			return fmt.Errorf("metric name '%s' doesn't match regex '%s'", currentMapping.Name, metricNameRE)
		}

		if currentMapping.MatchType == "" {
			currentMapping.MatchType = n.Defaults.MatchType
		}

		if currentMapping.Action == "" {
			currentMapping.Action = ActionTypeMap
		}

		if currentMapping.MatchType == MatchTypeGlob {
			n.doFSM = true
			if !metricLineRE.MatchString(currentMapping.Match) {
				return fmt.Errorf("invalid match: %s", currentMapping.Match)
			}

			captureCount := n.FSM.AddState(currentMapping.Match, string(currentMapping.MatchMetricType),
				remainingMappingsCount, currentMapping)

			currentMapping.nameFormatter = fsm.NewTemplateFormatter(currentMapping.Name, captureCount)

			labelKeys := make([]string, len(currentMapping.Labels))
			labelFormatters := make([]*fsm.TemplateFormatter, len(currentMapping.Labels))
			labelIndex := 0
			for label, valueExpr := range currentMapping.Labels {
				labelKeys[labelIndex] = label
				labelFormatters[labelIndex] = fsm.NewTemplateFormatter(valueExpr, captureCount)
				labelIndex++
			}
			currentMapping.labelFormatters = labelFormatters
			currentMapping.labelKeys = labelKeys
		} else {
			if regex, err := regexp.Compile(currentMapping.Match); err != nil {
				return fmt.Errorf("invalid regex %s in mapping: %v", currentMapping.Match, err)
			} else {
				currentMapping.regex = regex
			}
			n.doRegex = true
		}

		if currentMapping.ObserverType == "" {
			currentMapping.ObserverType = n.Defaults.ObserverType
		}

		if currentMapping.LegacyQuantiles != nil &&
			(currentMapping.SummaryOptions == nil || currentMapping.SummaryOptions.Quantiles != nil) {
			level.Warn(m.Logger).Log("msg", "using the top level quantiles is deprecated.  Please use quantiles in the summary_options hierarchy")
		}

		if currentMapping.LegacyBuckets != nil &&
			(currentMapping.HistogramOptions == nil || currentMapping.HistogramOptions.Buckets != nil) {
			level.Warn(m.Logger).Log("msg", "using the top level buckets is deprecated.  Please use buckets in the histogram_options hierarchy")
		}

		if currentMapping.SummaryOptions != nil &&
			currentMapping.LegacyQuantiles != nil &&
			currentMapping.SummaryOptions.Quantiles != nil {
			return fmt.Errorf("cannot use quantiles in both the top level and summary options at the same time in %s", currentMapping.Match)
		}

		if currentMapping.HistogramOptions != nil &&
			currentMapping.LegacyBuckets != nil &&
			currentMapping.HistogramOptions.Buckets != nil {
			return fmt.Errorf("cannot use buckets in both the top level and histogram options at the same time in %s", currentMapping.Match)
		}

		if currentMapping.ObserverType == ObserverTypeHistogram {
			if currentMapping.SummaryOptions != nil {
				return fmt.Errorf("cannot use histogram observer and summary options at the same time")
			}
			if currentMapping.HistogramOptions == nil {
				currentMapping.HistogramOptions = &HistogramOptions{}
			}
			if currentMapping.LegacyBuckets != nil && len(currentMapping.LegacyBuckets) != 0 {
				currentMapping.HistogramOptions.Buckets = currentMapping.LegacyBuckets
			}
			if currentMapping.HistogramOptions.Buckets == nil || len(currentMapping.HistogramOptions.Buckets) == 0 {
				currentMapping.HistogramOptions.Buckets = n.Defaults.HistogramOptions.Buckets
			}
		}

		if currentMapping.ObserverType == ObserverTypeSummary {
			if currentMapping.HistogramOptions != nil {
				return fmt.Errorf("cannot use summary observer and histogram options at the same time")
			}
			if currentMapping.SummaryOptions == nil {
				currentMapping.SummaryOptions = &SummaryOptions{}
			}
			if currentMapping.LegacyQuantiles != nil && len(currentMapping.LegacyQuantiles) != 0 {
				currentMapping.SummaryOptions.Quantiles = currentMapping.LegacyQuantiles
			}
			if currentMapping.SummaryOptions.Quantiles == nil || len(currentMapping.SummaryOptions.Quantiles) == 0 {
				currentMapping.SummaryOptions.Quantiles = n.Defaults.SummaryOptions.Quantiles
			}
			if currentMapping.SummaryOptions.MaxAge == 0 {
				currentMapping.SummaryOptions.MaxAge = n.Defaults.SummaryOptions.MaxAge
			}
			if currentMapping.SummaryOptions.AgeBuckets == 0 {
				currentMapping.SummaryOptions.AgeBuckets = n.Defaults.SummaryOptions.AgeBuckets
			}
			if currentMapping.SummaryOptions.BufCap == 0 {
				currentMapping.SummaryOptions.BufCap = n.Defaults.SummaryOptions.BufCap
			}
		}

		if currentMapping.Ttl == 0 && n.Defaults.Ttl > 0 {
			currentMapping.Ttl = n.Defaults.Ttl
		}
	}

	m.mutex.Lock()
	defer m.mutex.Unlock()

	if m.Logger == nil {
		m.Logger = log.NewNopLogger()
	}

	m.Defaults = n.Defaults
	m.Mappings = n.Mappings

	// Reset the cache since this function can be used to reload config
	if m.cache != nil {
		m.cache.Reset()
	}

	if n.doFSM {
		var mappings []string
		for _, mapping := range n.Mappings {
			if mapping.MatchType == MatchTypeGlob {
				mappings = append(mappings, mapping.Match)
			}
		}
		n.FSM.BacktrackingNeeded = fsm.TestIfNeedBacktracking(mappings, n.FSM.OrderingDisabled, m.Logger)

		m.FSM = n.FSM
		m.doRegex = n.doRegex
	}
	m.doFSM = n.doFSM

	if m.MappingsCount != nil {
		m.MappingsCount.Set(float64(len(n.Mappings)))
	}

	return nil
}

func (m *MetricMapper) InitFromFile(fileName string) error {
	mappingStr, err := ioutil.ReadFile(fileName)
	if err != nil {
		return err
	}

	return m.InitFromYAMLString(string(mappingStr))
}

// UseCache tells the mapper to use a cache that implements the MetricMapperCache interface.
// This cache MUST be thread-safe!
func (m *MetricMapper) UseCache(cache MetricMapperCache) {
	m.mutex.Lock()
	defer m.mutex.Unlock()
	m.cache = cache
}

func (m *MetricMapper) GetMapping(statsdMetric string, statsdMetricType MetricType) (*MetricMapping, prometheus.Labels, bool) {
	m.mutex.RLock()
	defer m.mutex.RUnlock()

	// only use a cache if one is present
	if m.cache != nil {
		result, cached := m.cache.Get(formatKey(statsdMetric, statsdMetricType))
		if cached {
			r := result.(MetricMapperCacheResult)
			return r.Mapping, r.Labels, r.Matched
		}
	}

	// glob matching
	if m.doFSM {
		finalState, captures := m.FSM.GetMapping(statsdMetric, string(statsdMetricType))
		if finalState != nil && finalState.Result != nil {
			v := finalState.Result.(*MetricMapping)
			result := copyMetricMapping(v)
			result.Name = result.nameFormatter.Format(captures)

			labels := prometheus.Labels{}
			for index, formatter := range result.labelFormatters {
				labels[result.labelKeys[index]] = formatter.Format(captures)
			}

			r := MetricMapperCacheResult{
				Mapping: result,
				Matched: true,
				Labels:  labels,
			}
			// add match to cache
			if m.cache != nil {
				m.cache.Add(formatKey(statsdMetric, statsdMetricType), r)
			}

			return result, labels, true
		} else if !m.doRegex {
			// if there's no regex match type, return immediately
			// Add miss to cache
			if m.cache != nil {
				m.cache.Add(formatKey(statsdMetric, statsdMetricType), MetricMapperCacheResult{})
			}
			return nil, nil, false
		}
	}

	// regex matching
	for _, mapping := range m.Mappings {
		// if a rule don't have regex matching type, the regex field is unset
		if mapping.regex == nil {
			continue
		}
		matches := mapping.regex.FindStringSubmatchIndex(statsdMetric)
		if len(matches) == 0 {
			continue
		}

		mapping.Name = string(mapping.regex.ExpandString(
			[]byte{},
			mapping.Name,
			statsdMetric,
			matches,
		))

		if mt := mapping.MatchMetricType; mt != "" && mt != statsdMetricType {
			continue
		}

		labels := prometheus.Labels{}
		for label, valueExpr := range mapping.Labels {
			value := mapping.regex.ExpandString([]byte{}, valueExpr, statsdMetric, matches)
			labels[label] = string(value)
		}

		r := MetricMapperCacheResult{
			Mapping: &mapping,
			Matched: true,
			Labels:  labels,
		}
		// Add Match to cache
		if m.cache != nil {
			m.cache.Add(formatKey(statsdMetric, statsdMetricType), r)
		}

		return &mapping, labels, true
	}

	// Add Miss to cache
	if m.cache != nil {
		m.cache.Add(formatKey(statsdMetric, statsdMetricType), MetricMapperCacheResult{})
	}
	return nil, nil, false
}

// make a shallow copy so that we do not overwrite name
// as multiple names can be matched by same mapping
func copyMetricMapping(in *MetricMapping) *MetricMapping {
	out := *in
	return &out
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package downloader_test

import (
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"time"

	"github.com/juju/errors"
	gitjujutesting "github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/downloader"
	"github.com/juju/juju/testing"
)

type DownloadSuite struct {
	testing.BaseSuite
	gitjujutesting.HTTPSuite
}

func (s *DownloadSuite) SetUpSuite(c *gc.C) {
	s.BaseSuite.SetUpSuite(c)
	s.HTTPSuite.SetUpSuite(c)
}

func (s *DownloadSuite) TearDownSuite(c *gc.C) {
	s.HTTPSuite.TearDownSuite(c)
	s.BaseSuite.TearDownSuite(c)
}

func (s *DownloadSuite) SetUpTest(c *gc.C) {
	s.BaseSuite.SetUpTest(c)
	s.HTTPSuite.SetUpTest(c)
}

func (s *DownloadSuite) TearDownTest(c *gc.C) {
	s.HTTPSuite.TearDownTest(c)
	s.BaseSuite.TearDownTest(c)
}

var _ = gc.Suite(&DownloadSuite{})

func (s *DownloadSuite) URL(c *gc.C, path string) *url.URL {
	urlStr := s.HTTPSuite.URL(path)
	URL, err := url.Parse(urlStr)
	c.Assert(err, jc.ErrorIsNil)
	return URL
}

func (s *DownloadSuite) testDownload(c *gc.C, hostnameVerification utils.SSLHostnameVerification) {
	tmp := c.MkDir()
	gitjujutesting.Server.Response(200, nil, []byte("archive"))
	d := downloader.StartDownload(
		downloader.Request{
			URL:       s.URL(c, "/archive.tgz"),
			TargetDir: tmp,
		},
		downloader.NewHTTPBlobOpener(hostnameVerification),
	)
	status := <-d.Done()
	defer os.Remove(status.File.Name())
	defer status.File.Close()
	c.Assert(status.Err, gc.IsNil)
	c.Assert(status.File, gc.NotNil)

	dir, _ := filepath.Split(status.File.Name())
	c.Assert(filepath.Clean(dir), gc.Equals, tmp)
	assertFileContents(c, status.File, "archive")
}

func (s *DownloadSuite) TestDownloadWithoutDisablingSSLHostnameVerification(c *gc.C) {
	s.testDownload(c, utils.VerifySSLHostnames)
}

func (s *DownloadSuite) TestDownloadWithDisablingSSLHostnameVerification(c *gc.C) {
	s.testDownload(c, utils.NoVerifySSLHostnames)
}

func (s *DownloadSuite) TestDownloadError(c *gc.C) {
	gitjujutesting.Server.Response(404, nil, nil)
	d := downloader.StartDownload(
		downloader.Request{
			URL:       s.URL(c, "/archive.tgz"),
			TargetDir: c.MkDir(),
		},
		downloader.NewHTTPBlobOpener(utils.VerifySSLHostnames),
	)
	status := <-d.Done()
	c.Assert(status.File, gc.IsNil)
	c.Assert(status.Err, gc.ErrorMatches, `cannot download ".*": bad http response: 404 Not Found`)
}

func (s *DownloadSuite) TestStop(c *gc.C) {
	tmp := c.MkDir()
	d := downloader.StartDownload(
		downloader.Request{
			URL:       s.URL(c, "/x.tgz"),
			TargetDir: tmp,
		},
		downloader.NewHTTPBlobOpener(utils.VerifySSLHostnames),
	)
	d.Stop()
	select {
	case status := <-d.Done():
		c.Fatalf("received status %#v after stop", status)
	case <-time.After(testing.ShortWait):
	}
	infos, err := ioutil.ReadDir(tmp)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(infos, gc.HasLen, 0)
}

func (s *DownloadSuite) TestVerifyValid(c *gc.C) {
	stub := &gitjujutesting.Stub{}
	tmp := c.MkDir()
	gitjujutesting.Server.Response(200, nil, []byte("archive"))
	dl := downloader.StartDownload(
		downloader.Request{
			URL:       s.URL(c, "/archive.tgz"),
			TargetDir: tmp,
			Verify: func(f *os.File) error {
				stub.AddCall("Verify", f)
				return nil
			},
		},
		downloader.NewHTTPBlobOpener(utils.VerifySSLHostnames),
	)
	status := <-dl.Done()
	c.Assert(status.Err, jc.ErrorIsNil)

	stub.CheckCallNames(c, "Verify")
	stub.CheckCall(c, 0, "Verify", status.File)
}

func (s *DownloadSuite) TestVerifyInvalid(c *gc.C) {
	stub := &gitjujutesting.Stub{}
	tmp := c.MkDir()
	gitjujutesting.Server.Response(200, nil, []byte("archive"))
	invalid := errors.NotValidf("oops")
	dl := downloader.StartDownload(
		downloader.Request{
			URL:       s.URL(c, "/archive.tgz"),
			TargetDir: tmp,
			Verify: func(f *os.File) error {
				stub.AddCall("Verify", f)
				return invalid
			},
		},
		downloader.NewHTTPBlobOpener(utils.VerifySSLHostnames),
	)
	status := <-dl.Done()

	c.Check(errors.Cause(status.Err), gc.Equals, invalid)
	stub.CheckCallNames(c, "Verify")
	stub.CheckCall(c, 0, "Verify", status.File)
}

func assertFileContents(c *gc.C, f *os.File, expect string) {
	got, err := ioutil.ReadAll(f)
	c.Assert(err, jc.ErrorIsNil)
	if !c.Check(string(got), gc.Equals, expect) {
		info, err := f.Stat()
		c.Assert(err, jc.ErrorIsNil)
		c.Logf("info %#v", info)
	}
}
<commit_msg>Drop a superflous os.Remove().<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package downloader_test

import (
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"time"

	"github.com/juju/errors"
	gitjujutesting "github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/downloader"
	"github.com/juju/juju/testing"
)

type DownloadSuite struct {
	testing.BaseSuite
	gitjujutesting.HTTPSuite
}

func (s *DownloadSuite) SetUpSuite(c *gc.C) {
	s.BaseSuite.SetUpSuite(c)
	s.HTTPSuite.SetUpSuite(c)
}

func (s *DownloadSuite) TearDownSuite(c *gc.C) {
	s.HTTPSuite.TearDownSuite(c)
	s.BaseSuite.TearDownSuite(c)
}

func (s *DownloadSuite) SetUpTest(c *gc.C) {
	s.BaseSuite.SetUpTest(c)
	s.HTTPSuite.SetUpTest(c)
}

func (s *DownloadSuite) TearDownTest(c *gc.C) {
	s.HTTPSuite.TearDownTest(c)
	s.BaseSuite.TearDownTest(c)
}

var _ = gc.Suite(&DownloadSuite{})

func (s *DownloadSuite) URL(c *gc.C, path string) *url.URL {
	urlStr := s.HTTPSuite.URL(path)
	URL, err := url.Parse(urlStr)
	c.Assert(err, jc.ErrorIsNil)
	return URL
}

func (s *DownloadSuite) testDownload(c *gc.C, hostnameVerification utils.SSLHostnameVerification) {
	tmp := c.MkDir()
	gitjujutesting.Server.Response(200, nil, []byte("archive"))
	d := downloader.StartDownload(
		downloader.Request{
			URL:       s.URL(c, "/archive.tgz"),
			TargetDir: tmp,
		},
		downloader.NewHTTPBlobOpener(hostnameVerification),
	)
	status := <-d.Done()
	defer status.File.Close()
	c.Assert(status.Err, gc.IsNil)
	c.Assert(status.File, gc.NotNil)

	dir, _ := filepath.Split(status.File.Name())
	c.Assert(filepath.Clean(dir), gc.Equals, tmp)
	assertFileContents(c, status.File, "archive")
}

func (s *DownloadSuite) TestDownloadWithoutDisablingSSLHostnameVerification(c *gc.C) {
	s.testDownload(c, utils.VerifySSLHostnames)
}

func (s *DownloadSuite) TestDownloadWithDisablingSSLHostnameVerification(c *gc.C) {
	s.testDownload(c, utils.NoVerifySSLHostnames)
}

func (s *DownloadSuite) TestDownloadError(c *gc.C) {
	gitjujutesting.Server.Response(404, nil, nil)
	d := downloader.StartDownload(
		downloader.Request{
			URL:       s.URL(c, "/archive.tgz"),
			TargetDir: c.MkDir(),
		},
		downloader.NewHTTPBlobOpener(utils.VerifySSLHostnames),
	)
	status := <-d.Done()
	c.Assert(status.File, gc.IsNil)
	c.Assert(status.Err, gc.ErrorMatches, `cannot download ".*": bad http response: 404 Not Found`)
}

func (s *DownloadSuite) TestStop(c *gc.C) {
	tmp := c.MkDir()
	d := downloader.StartDownload(
		downloader.Request{
			URL:       s.URL(c, "/x.tgz"),
			TargetDir: tmp,
		},
		downloader.NewHTTPBlobOpener(utils.VerifySSLHostnames),
	)
	d.Stop()
	select {
	case status := <-d.Done():
		c.Fatalf("received status %#v after stop", status)
	case <-time.After(testing.ShortWait):
	}
	infos, err := ioutil.ReadDir(tmp)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(infos, gc.HasLen, 0)
}

func (s *DownloadSuite) TestVerifyValid(c *gc.C) {
	stub := &gitjujutesting.Stub{}
	tmp := c.MkDir()
	gitjujutesting.Server.Response(200, nil, []byte("archive"))
	dl := downloader.StartDownload(
		downloader.Request{
			URL:       s.URL(c, "/archive.tgz"),
			TargetDir: tmp,
			Verify: func(f *os.File) error {
				stub.AddCall("Verify", f)
				return nil
			},
		},
		downloader.NewHTTPBlobOpener(utils.VerifySSLHostnames),
	)
	status := <-dl.Done()
	c.Assert(status.Err, jc.ErrorIsNil)

	stub.CheckCallNames(c, "Verify")
	stub.CheckCall(c, 0, "Verify", status.File)
}

func (s *DownloadSuite) TestVerifyInvalid(c *gc.C) {
	stub := &gitjujutesting.Stub{}
	tmp := c.MkDir()
	gitjujutesting.Server.Response(200, nil, []byte("archive"))
	invalid := errors.NotValidf("oops")
	dl := downloader.StartDownload(
		downloader.Request{
			URL:       s.URL(c, "/archive.tgz"),
			TargetDir: tmp,
			Verify: func(f *os.File) error {
				stub.AddCall("Verify", f)
				return invalid
			},
		},
		downloader.NewHTTPBlobOpener(utils.VerifySSLHostnames),
	)
	status := <-dl.Done()

	c.Check(errors.Cause(status.Err), gc.Equals, invalid)
	stub.CheckCallNames(c, "Verify")
	stub.CheckCall(c, 0, "Verify", status.File)
}

func assertFileContents(c *gc.C, f *os.File, expect string) {
	got, err := ioutil.ReadAll(f)
	c.Assert(err, jc.ErrorIsNil)
	if !c.Check(string(got), gc.Equals, expect) {
		info, err := f.Stat()
		c.Assert(err, jc.ErrorIsNil)
		c.Logf("info %#v", info)
	}
}
<|endoftext|>


<commit_before>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package plugins

import (
	"net/http"
	"regexp"
	"strconv"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/accounts"
	"github.com/sapcc/limes/pkg/limes"
	"github.com/sapcc/limes/pkg/util"
)

type swiftPlugin struct{}

var swiftResources = []limes.ResourceInfo{
	limes.ResourceInfo{
		Name: "capacity",
		Unit: limes.UnitBytes,
	},
}

//TODO Make Auth prefix configurable
var urlRegex = regexp.MustCompile("(v1/AUTH_)[a-zA-Z0-9]+")

func init() {
	limes.RegisterQuotaPlugin(&swiftPlugin{})
}

//ServiceType implements the limes.Plugin interface.
func (p *swiftPlugin) ServiceType() string {
	return "object-store"
}

//Resources implements the limes.Plugin interface.
func (p *swiftPlugin) Resources() []limes.ResourceInfo {
	return swiftResources
}

func (p *swiftPlugin) Client(driver limes.Driver) (*gophercloud.ServiceClient, error) {
	return openstack.NewObjectStorageV1(driver.Client(),
		gophercloud.EndpointOpts{Availability: gophercloud.AvailabilityPublic},
	)
}

//Scrape implements the limes.Plugin interface.
func (p *swiftPlugin) Scrape(driver limes.Driver, domainUUID, projectUUID string) (map[string]limes.ResourceData, error) {
	client, err := p.projectClient(driver, projectUUID)
	if err != nil {
		return nil, err
	}

	//Get account metadata
	account := getAccount(client, projectUUID)
	if account == nil {
		//Swift account does not exist, but the keystone project
		return map[string]limes.ResourceData{
			"capacity": limes.ResourceData{
				Quota: 0,
				Usage: 0,
			},
		}, nil
	} else if account.Err != nil {
		return nil, err
	}

	//Extract quota, if set
	var quota int64 = -1
	quotaHeader := account.Header.Get("X-Account-Meta-Quota-Bytes")
	if quotaHeader != "" {
		quota, _ = strconv.ParseInt(quotaHeader, 10, 64)
	}

	//Extract usage
	var usage int64
	usageHeader := account.Header.Get("X-Account-Bytes-Used")
	if usageHeader != "" {
		usage, _ = strconv.ParseInt(usageHeader, 10, 64)
	}

	util.LogDebug("Swift Account %s: quota '%d' - usage '%d'", projectUUID, quota, usage)
	return map[string]limes.ResourceData{
		"capacity": limes.ResourceData{
			Quota: quota,
			Usage: uint64(usage),
		},
	}, nil
}

//SetQuota implements the limes.Plugin interface.
func (p *swiftPlugin) SetQuota(driver limes.Driver, domainUUID, projectUUID string, quotas map[string]uint64) error {
	client, err := p.projectClient(driver, projectUUID)
	if err != nil {
		return err
	}

	headers := map[string]string{
		"X-Account-Meta-Quota-Bytes": string(quotas["capacity"]),
		//this header brought to you by https://github.com/sapcc/swift-addons
		"X-Account-Project-Domain-Id-Override": domainUUID,
	}

	result, err := updateAccount(client, headers)
	if result.StatusCode == http.StatusNotFound && quotas["capacity"] > 0 {
		//account does not exist yet - if there is a non-zero quota, enable it now
		_, err = putAccount(client, headers)
		if err != nil {
			util.LogInfo("Swift Account %s created", projectUUID)
		}
	}
	return err
}

//Get the project scoped cliet with specific storage URL
func (p *swiftPlugin) projectClient(driver limes.Driver, projectUUID string) (*gophercloud.ServiceClient, error) {
	client, err := p.Client(driver)
	if err != nil {
		return nil, err
	}

	//We act as Reseller_Admin here, but cannot use the endpoint url returned from catalogue
	//Replace the resellers project id with the requested one
	client.Endpoint = urlRegex.ReplaceAllString(client.Endpoint, "${1}"+projectUUID)
	util.LogDebug(client.Endpoint)
	return client, nil
}

//Wrapping the accounts.Get because the swift account might not be created if account_auto_create = false
func getAccount(client *gophercloud.ServiceClient, projectUUID string) *accounts.GetResult {
	//Get account metadata
	var result accounts.GetResult
	result = accounts.Get(client, accounts.GetOpts{})
	if _, ok := result.Err.(gophercloud.ErrDefault404); ok {
		//Swift Account does not exist. This is expected esp. if account_auto_create is disabled
		util.LogDebug("Swift Account %s does not exist", projectUUID)
		return nil
	}
	return &result
}

//Issue a POST request to the account with own headers
func updateAccount(c *gophercloud.ServiceClient, headers map[string]string) (*http.Response, error) {
	return c.Request("POST", c.Endpoint, &gophercloud.RequestOpts{
		MoreHeaders: headers,
		OkCodes:     []int{200, 204},
	})
}

//Issue a PUT request to the account with own headers
func putAccount(c *gophercloud.ServiceClient, headers map[string]string) (*http.Response, error) {
	return c.Request("PUT", c.Endpoint, &gophercloud.RequestOpts{
		MoreHeaders: headers,
		OkCodes:     []int{201},
	})
}
<commit_msg>fix cast of uint to string<commit_after>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package plugins

import (
	"net/http"
	"regexp"
	"strconv"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/objectstorage/v1/accounts"
	"github.com/sapcc/limes/pkg/limes"
	"github.com/sapcc/limes/pkg/util"
)

type swiftPlugin struct{}

var swiftResources = []limes.ResourceInfo{
	limes.ResourceInfo{
		Name: "capacity",
		Unit: limes.UnitBytes,
	},
}

//TODO Make Auth prefix configurable
var urlRegex = regexp.MustCompile("(v1/AUTH_)[a-zA-Z0-9]+")

func init() {
	limes.RegisterQuotaPlugin(&swiftPlugin{})
}

//ServiceType implements the limes.Plugin interface.
func (p *swiftPlugin) ServiceType() string {
	return "object-store"
}

//Resources implements the limes.Plugin interface.
func (p *swiftPlugin) Resources() []limes.ResourceInfo {
	return swiftResources
}

func (p *swiftPlugin) Client(driver limes.Driver) (*gophercloud.ServiceClient, error) {
	return openstack.NewObjectStorageV1(driver.Client(),
		gophercloud.EndpointOpts{Availability: gophercloud.AvailabilityPublic},
	)
}

//Scrape implements the limes.Plugin interface.
func (p *swiftPlugin) Scrape(driver limes.Driver, domainUUID, projectUUID string) (map[string]limes.ResourceData, error) {
	client, err := p.projectClient(driver, projectUUID)
	if err != nil {
		return nil, err
	}

	//Get account metadata
	account := getAccount(client, projectUUID)
	if account == nil {
		//Swift account does not exist, but the keystone project
		return map[string]limes.ResourceData{
			"capacity": limes.ResourceData{
				Quota: 0,
				Usage: 0,
			},
		}, nil
	} else if account.Err != nil {
		return nil, err
	}

	//Extract quota, if set
	var quota int64 = -1
	quotaHeader := account.Header.Get("X-Account-Meta-Quota-Bytes")
	if quotaHeader != "" {
		quota, _ = strconv.ParseInt(quotaHeader, 10, 64)
	}

	//Extract usage
	var usage int64
	usageHeader := account.Header.Get("X-Account-Bytes-Used")
	if usageHeader != "" {
		usage, _ = strconv.ParseInt(usageHeader, 10, 64)
	}

	util.LogDebug("Swift Account %s: quota '%d' - usage '%d'", projectUUID, quota, usage)
	return map[string]limes.ResourceData{
		"capacity": limes.ResourceData{
			Quota: quota,
			Usage: uint64(usage),
		},
	}, nil
}

//SetQuota implements the limes.Plugin interface.
func (p *swiftPlugin) SetQuota(driver limes.Driver, domainUUID, projectUUID string, quotas map[string]uint64) error {
	client, err := p.projectClient(driver, projectUUID)
	if err != nil {
		return err
	}

	headers := map[string]string{
		"X-Account-Meta-Quota-Bytes": strconv.FormatUint(quotas["capacity"], 10),
		//this header brought to you by https://github.com/sapcc/swift-addons
		"X-Account-Project-Domain-Id-Override": domainUUID,
	}

	result, err := updateAccount(client, headers)
	if result.StatusCode == http.StatusNotFound && quotas["capacity"] > 0 {
		//account does not exist yet - if there is a non-zero quota, enable it now
		_, err = putAccount(client, headers)
		if err != nil {
			util.LogInfo("Swift Account %s created", projectUUID)
		}
	}
	return err
}

//Get the project scoped cliet with specific storage URL
func (p *swiftPlugin) projectClient(driver limes.Driver, projectUUID string) (*gophercloud.ServiceClient, error) {
	client, err := p.Client(driver)
	if err != nil {
		return nil, err
	}

	//We act as Reseller_Admin here, but cannot use the endpoint url returned from catalogue
	//Replace the resellers project id with the requested one
	client.Endpoint = urlRegex.ReplaceAllString(client.Endpoint, "${1}"+projectUUID)
	util.LogDebug(client.Endpoint)
	return client, nil
}

//Wrapping the accounts.Get because the swift account might not be created if account_auto_create = false
func getAccount(client *gophercloud.ServiceClient, projectUUID string) *accounts.GetResult {
	//Get account metadata
	var result accounts.GetResult
	result = accounts.Get(client, accounts.GetOpts{})
	if _, ok := result.Err.(gophercloud.ErrDefault404); ok {
		//Swift Account does not exist. This is expected esp. if account_auto_create is disabled
		util.LogDebug("Swift Account %s does not exist", projectUUID)
		return nil
	}
	return &result
}

//Issue a POST request to the account with own headers
func updateAccount(c *gophercloud.ServiceClient, headers map[string]string) (*http.Response, error) {
	return c.Request("POST", c.Endpoint, &gophercloud.RequestOpts{
		MoreHeaders: headers,
		OkCodes:     []int{200, 204},
	})
}

//Issue a PUT request to the account with own headers
func putAccount(c *gophercloud.ServiceClient, headers map[string]string) (*http.Response, error) {
	return c.Request("PUT", c.Endpoint, &gophercloud.RequestOpts{
		MoreHeaders: headers,
		OkCodes:     []int{201},
	})
}
<|endoftext|>


<commit_before>package schema

import "reflect"

type Thing struct {
	SchemaType    string `json:"@type"`
	SchemaContext string `json:"@context"`
}

var (
	pkgPath string
)

func Fill(thing interface{}) interface{} {
	v := reflect.ValueOf(thing)
	if v.Kind() != reflect.Ptr {
		panic("schema.Fill: expects struct pointer")
	}

	v = reflect.Indirect(v)
	t := v.Type()
	if t.Kind() != reflect.Struct {
		return thing
	}

	vthing := v.FieldByName("Thing")
	if !vthing.IsValid() {
		return thing
	}

	if vthing.Kind() == reflect.Ptr {
		vthing = reflect.Indirect(vthing)
		if !vthing.IsValid() {
			vthing = reflect.ValueOf(&Thing{})
			v.FieldByName("Thing").Set(vthing)
			vthing = reflect.Indirect(vthing)
		}
	}
	vtype := vthing.FieldByName("SchemaType")
	if vtype.String() != "" {
		return thing
	}

	if path := t.PkgPath(); path != pkgPath {
		fthing, _ := t.FieldByName("Thing")
		tag := fthing.Tag.Get("schema")
		if tag != "" {
			vtype.SetString(tag)
		} else {
			vtype.SetString("http://" + path + "/" + t.Name())
		}
	} else {
		vtype.SetString(t.Name())
	}

	return thing
}

func init() {
	pkgPath = reflect.TypeOf(Thing{}).PkgPath()
}
<commit_msg>Schema: More JSON-LD shortcuts.<commit_after>package schema

import "reflect"

type Thing struct {
	SchemaContext string `json:"@context,omitempty"`
	SchemaType    string `json:"@type,omitempty"`
	SchemaId      string `json:"@id,omitempty"`
	SchemaLabel   string `json:"rdfs:label,omitempty"`
}

var (
	pkgPath string
)

func Fill(thing interface{}) interface{} {
	v := reflect.ValueOf(thing)
	if v.Kind() != reflect.Ptr {
		panic("schema.Fill: expects struct pointer")
	}

	v = reflect.Indirect(v)
	t := v.Type()
	if t.Kind() != reflect.Struct {
		return thing
	}

	vthing := v.FieldByName("Thing")
	if !vthing.IsValid() {
		return thing
	}

	if vthing.Kind() == reflect.Ptr {
		vthing = reflect.Indirect(vthing)
		if !vthing.IsValid() {
			vthing = reflect.ValueOf(&Thing{})
			v.FieldByName("Thing").Set(vthing)
			vthing = reflect.Indirect(vthing)
		}
	}
	vtype := vthing.FieldByName("SchemaType")
	if vtype.String() != "" {
		return thing
	}

	if path := t.PkgPath(); path != pkgPath {
		fthing, _ := t.FieldByName("Thing")
		tag := fthing.Tag.Get("schema")
		if tag != "" {
			vtype.SetString(tag)
		} else {
			vtype.SetString("http://" + path + "/" + t.Name())
		}
	} else {
		vtype.SetString(t.Name())
	}

	return thing
}

func init() {
	pkgPath = reflect.TypeOf(Thing{}).PkgPath()
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"strings"

	hooks "github.com/appscode/kubernetes-webhook-util/admission/v1beta1"
	admissionreview "github.com/appscode/kubernetes-webhook-util/registry/admissionreview/v1beta1"
	"github.com/appscode/searchlight/apis/incidents"
	"github.com/appscode/searchlight/apis/incidents/install"
	"github.com/appscode/searchlight/apis/incidents/v1alpha1"
	"github.com/appscode/searchlight/pkg/operator"
	ackregistry "github.com/appscode/searchlight/pkg/registry/acknowledgement"
	admission "k8s.io/api/admission/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/apimachinery/pkg/version"
	"k8s.io/apiserver/pkg/registry/rest"
	genericapiserver "k8s.io/apiserver/pkg/server"
)

var (
	Scheme = runtime.NewScheme()
	Codecs = serializer.NewCodecFactory(Scheme)
)

func init() {
	install.Install(Scheme)
	admission.AddToScheme(Scheme)

	// we need to add the options to empty v1
	// TODO fix the server code to avoid this
	metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})

	// TODO: keep the generic API server from wanting this
	unversioned := schema.GroupVersion{Group: "", Version: "v1"}
	Scheme.AddUnversionedTypes(unversioned,
		&metav1.Status{},
		&metav1.APIVersions{},
		&metav1.APIGroupList{},
		&metav1.APIGroup{},
		&metav1.APIResourceList{},
	)
}

type SearchlightConfig struct {
	GenericConfig  *genericapiserver.RecommendedConfig
	OperatorConfig *operator.OperatorConfig
}

// SearchlightServer contains state for a Kubernetes cluster master/api server.
type SearchlightServer struct {
	GenericAPIServer *genericapiserver.GenericAPIServer
	Operator         *operator.Operator
}

func (op *SearchlightServer) Run(stopCh <-chan struct{}) error {
	go op.Operator.Run(stopCh)
	return op.GenericAPIServer.PrepareRun().Run(stopCh)
}

type completedConfig struct {
	GenericConfig  genericapiserver.CompletedConfig
	OperatorConfig *operator.OperatorConfig
}

type CompletedConfig struct {
	// Embed a private pointer that cannot be instantiated outside of this package.
	*completedConfig
}

// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
func (c *SearchlightConfig) Complete() CompletedConfig {
	completedCfg := completedConfig{
		c.GenericConfig.Complete(),
		c.OperatorConfig,
	}

	completedCfg.GenericConfig.Version = &version.Info{
		Major: "1",
		Minor: "1",
	}

	return CompletedConfig{&completedCfg}
}

// New returns a new instance of SearchlightServer from the given config.
func (c completedConfig) New() (*SearchlightServer, error) {
	genericServer, err := c.GenericConfig.New("searchlight-server", genericapiserver.NewEmptyDelegate()) // completion is done in Complete, no need for a second time
	if err != nil {
		return nil, err
	}
	ctrl, err := c.OperatorConfig.New()
	if err != nil {
		return nil, err
	}

	s := &SearchlightServer{
		GenericAPIServer: genericServer,
		Operator:         ctrl,
	}

	for _, versionMap := range admissionHooksByGroupThenVersion(c.OperatorConfig.AdmissionHooks...) {
		// TODO we're going to need a later k8s.io/apiserver so that we can get discovery to list a different group version for
		// our endpoint which we'll use to back some custom storage which will consume the AdmissionReview type and give back the correct response
		apiGroupInfo := genericapiserver.APIGroupInfo{
			PrioritizedVersions:          []schema.GroupVersion{admission.SchemeGroupVersion},
			VersionedResourcesStorageMap: map[string]map[string]rest.Storage{},
			// TODO unhardcode this.  It was hardcoded before, but we need to re-evaluate
			OptionsExternalVersion: &schema.GroupVersion{Version: "v1"},
			Scheme:                 Scheme,
			ParameterCodec:         metav1.ParameterCodec,
			NegotiatedSerializer:   Codecs,
		}

		for _, admissionHooks := range versionMap {
			for i := range admissionHooks {
				admissionHook := admissionHooks[i]
				admissionResource, _ := admissionHook.Resource()
				admissionVersion := admissionResource.GroupVersion()

				// just overwrite the groupversion with a random one.  We don't really care or know.
				apiGroupInfo.PrioritizedVersions = appendUniqueGroupVersion(apiGroupInfo.PrioritizedVersions, admissionVersion)

				admissionReview := admissionreview.NewREST(admissionHook.Admit)
				v1alpha1storage, ok := apiGroupInfo.VersionedResourcesStorageMap[admissionVersion.Version]
				if !ok {
					v1alpha1storage = map[string]rest.Storage{}
				}
				v1alpha1storage[admissionResource.Resource] = admissionReview
				apiGroupInfo.VersionedResourcesStorageMap[admissionVersion.Version] = v1alpha1storage
			}
		}

		if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil {
			return nil, err
		}
	}

	for i := range c.OperatorConfig.AdmissionHooks {
		admissionHook := c.OperatorConfig.AdmissionHooks[i]
		postStartName := postStartHookName(admissionHook)
		if len(postStartName) == 0 {
			continue
		}
		s.GenericAPIServer.AddPostStartHookOrDie(postStartName,
			func(context genericapiserver.PostStartHookContext) error {
				return admissionHook.Initialize(c.OperatorConfig.ClientConfig, context.StopCh)
			},
		)
	}

	{
		apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(incidents.GroupName, Scheme, metav1.ParameterCodec, Codecs)
		v1alpha1storage := map[string]rest.Storage{}
		v1alpha1storage[v1alpha1.ResourcePluralAcknowledgement] = ackregistry.NewREST(c.OperatorConfig.ClientConfig, c.OperatorConfig.IcingaClient)
		apiGroupInfo.VersionedResourcesStorageMap["v1alpha1"] = v1alpha1storage

		if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil {
			return nil, err
		}
	}

	return s, nil
}

func appendUniqueGroupVersion(slice []schema.GroupVersion, elems ...schema.GroupVersion) []schema.GroupVersion {
	m := map[schema.GroupVersion]bool{}
	for _, gv := range slice {
		m[gv] = true
	}
	for _, e := range elems {
		m[e] = true
	}
	out := make([]schema.GroupVersion, 0, len(m))
	for gv := range m {
		out = append(out, gv)
	}
	return out
}

func postStartHookName(hook hooks.AdmissionHook) string {
	var ns []string
	gvr, _ := hook.Resource()
	ns = append(ns, fmt.Sprintf("admit-%s.%s.%s", gvr.Resource, gvr.Version, gvr.Group))
	if len(ns) == 0 {
		return ""
	}
	return strings.Join(append(ns, "init"), "-")
}

func admissionHooksByGroupThenVersion(admissionHooks ...hooks.AdmissionHook) map[string]map[string][]hooks.AdmissionHook {
	ret := map[string]map[string][]hooks.AdmissionHook{}

	for i := range admissionHooks {
		hook := admissionHooks[i]
		gvr, _ := hook.Resource()
		group, ok := ret[gvr.Group]
		if !ok {
			group = map[string][]hooks.AdmissionHook{}
			ret[gvr.Group] = group
		}
		group[gvr.Version] = append(group[gvr.Version], hook)
	}
	return ret
}
<commit_msg>Don't add admission/v1beta1 group as a prioritized version (#395)<commit_after>package server

import (
	"fmt"
	"strings"

	hooks "github.com/appscode/kubernetes-webhook-util/admission/v1beta1"
	admissionreview "github.com/appscode/kubernetes-webhook-util/registry/admissionreview/v1beta1"
	"github.com/appscode/searchlight/apis/incidents"
	"github.com/appscode/searchlight/apis/incidents/install"
	"github.com/appscode/searchlight/apis/incidents/v1alpha1"
	"github.com/appscode/searchlight/pkg/operator"
	ackregistry "github.com/appscode/searchlight/pkg/registry/acknowledgement"
	admission "k8s.io/api/admission/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/apimachinery/pkg/version"
	"k8s.io/apiserver/pkg/registry/rest"
	genericapiserver "k8s.io/apiserver/pkg/server"
)

var (
	Scheme = runtime.NewScheme()
	Codecs = serializer.NewCodecFactory(Scheme)
)

func init() {
	install.Install(Scheme)
	admission.AddToScheme(Scheme)

	// we need to add the options to empty v1
	// TODO fix the server code to avoid this
	metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})

	// TODO: keep the generic API server from wanting this
	unversioned := schema.GroupVersion{Group: "", Version: "v1"}
	Scheme.AddUnversionedTypes(unversioned,
		&metav1.Status{},
		&metav1.APIVersions{},
		&metav1.APIGroupList{},
		&metav1.APIGroup{},
		&metav1.APIResourceList{},
	)
}

type SearchlightConfig struct {
	GenericConfig  *genericapiserver.RecommendedConfig
	OperatorConfig *operator.OperatorConfig
}

// SearchlightServer contains state for a Kubernetes cluster master/api server.
type SearchlightServer struct {
	GenericAPIServer *genericapiserver.GenericAPIServer
	Operator         *operator.Operator
}

func (op *SearchlightServer) Run(stopCh <-chan struct{}) error {
	go op.Operator.Run(stopCh)
	return op.GenericAPIServer.PrepareRun().Run(stopCh)
}

type completedConfig struct {
	GenericConfig  genericapiserver.CompletedConfig
	OperatorConfig *operator.OperatorConfig
}

type CompletedConfig struct {
	// Embed a private pointer that cannot be instantiated outside of this package.
	*completedConfig
}

// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
func (c *SearchlightConfig) Complete() CompletedConfig {
	completedCfg := completedConfig{
		c.GenericConfig.Complete(),
		c.OperatorConfig,
	}

	completedCfg.GenericConfig.Version = &version.Info{
		Major: "1",
		Minor: "1",
	}

	return CompletedConfig{&completedCfg}
}

// New returns a new instance of SearchlightServer from the given config.
func (c completedConfig) New() (*SearchlightServer, error) {
	genericServer, err := c.GenericConfig.New("searchlight-server", genericapiserver.NewEmptyDelegate()) // completion is done in Complete, no need for a second time
	if err != nil {
		return nil, err
	}
	ctrl, err := c.OperatorConfig.New()
	if err != nil {
		return nil, err
	}

	s := &SearchlightServer{
		GenericAPIServer: genericServer,
		Operator:         ctrl,
	}

	for _, versionMap := range admissionHooksByGroupThenVersion(c.OperatorConfig.AdmissionHooks...) {
		// TODO we're going to need a later k8s.io/apiserver so that we can get discovery to list a different group version for
		// our endpoint which we'll use to back some custom storage which will consume the AdmissionReview type and give back the correct response
		apiGroupInfo := genericapiserver.APIGroupInfo{
			VersionedResourcesStorageMap: map[string]map[string]rest.Storage{},
			// TODO unhardcode this.  It was hardcoded before, but we need to re-evaluate
			OptionsExternalVersion: &schema.GroupVersion{Version: "v1"},
			Scheme:                 Scheme,
			ParameterCodec:         metav1.ParameterCodec,
			NegotiatedSerializer:   Codecs,
		}

		for _, admissionHooks := range versionMap {
			for i := range admissionHooks {
				admissionHook := admissionHooks[i]
				admissionResource, _ := admissionHook.Resource()
				admissionVersion := admissionResource.GroupVersion()

				// just overwrite the groupversion with a random one.  We don't really care or know.
				apiGroupInfo.PrioritizedVersions = appendUniqueGroupVersion(apiGroupInfo.PrioritizedVersions, admissionVersion)

				admissionReview := admissionreview.NewREST(admissionHook.Admit)
				v1alpha1storage, ok := apiGroupInfo.VersionedResourcesStorageMap[admissionVersion.Version]
				if !ok {
					v1alpha1storage = map[string]rest.Storage{}
				}
				v1alpha1storage[admissionResource.Resource] = admissionReview
				apiGroupInfo.VersionedResourcesStorageMap[admissionVersion.Version] = v1alpha1storage
			}
		}

		if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil {
			return nil, err
		}
	}

	for i := range c.OperatorConfig.AdmissionHooks {
		admissionHook := c.OperatorConfig.AdmissionHooks[i]
		postStartName := postStartHookName(admissionHook)
		if len(postStartName) == 0 {
			continue
		}
		s.GenericAPIServer.AddPostStartHookOrDie(postStartName,
			func(context genericapiserver.PostStartHookContext) error {
				return admissionHook.Initialize(c.OperatorConfig.ClientConfig, context.StopCh)
			},
		)
	}

	{
		apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(incidents.GroupName, Scheme, metav1.ParameterCodec, Codecs)
		v1alpha1storage := map[string]rest.Storage{}
		v1alpha1storage[v1alpha1.ResourcePluralAcknowledgement] = ackregistry.NewREST(c.OperatorConfig.ClientConfig, c.OperatorConfig.IcingaClient)
		apiGroupInfo.VersionedResourcesStorageMap["v1alpha1"] = v1alpha1storage

		if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil {
			return nil, err
		}
	}

	return s, nil
}

func appendUniqueGroupVersion(slice []schema.GroupVersion, elems ...schema.GroupVersion) []schema.GroupVersion {
	m := map[schema.GroupVersion]bool{}
	for _, gv := range slice {
		m[gv] = true
	}
	for _, e := range elems {
		m[e] = true
	}
	out := make([]schema.GroupVersion, 0, len(m))
	for gv := range m {
		out = append(out, gv)
	}
	return out
}

func postStartHookName(hook hooks.AdmissionHook) string {
	var ns []string
	gvr, _ := hook.Resource()
	ns = append(ns, fmt.Sprintf("admit-%s.%s.%s", gvr.Resource, gvr.Version, gvr.Group))
	if len(ns) == 0 {
		return ""
	}
	return strings.Join(append(ns, "init"), "-")
}

func admissionHooksByGroupThenVersion(admissionHooks ...hooks.AdmissionHook) map[string]map[string][]hooks.AdmissionHook {
	ret := map[string]map[string][]hooks.AdmissionHook{}

	for i := range admissionHooks {
		hook := admissionHooks[i]
		gvr, _ := hook.Resource()
		group, ok := ret[gvr.Group]
		if !ok {
			group = map[string][]hooks.AdmissionHook{}
			ret[gvr.Group] = group
		}
		group[gvr.Version] = append(group[gvr.Version], hook)
	}
	return ret
}
<|endoftext|>


<commit_before>// Copyright © 2012 Hraban Luyat <hraban@0brg.net>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.

// Light-weight in-memory LRU (object) cache library for Go.
//
// To use this library, first create a cache:
//
//      c := lrucache.New(1234)
//
// Then define a type that implements the Cacheable interface:
//
//      type cacheableInt int
//
//      func (i cacheableInt) OnPurge(deleted bool) {
//          fmt.Printf("Purging %d\n", i)
//      }
//
//      func (i cacheableInt) Size() int64 {
//          return 1
//      }
//
// Finally:
//
//     for i := 0; i < 2000; i++ {
//         c.Set(strconv.Itoa(i), cacheableInt(i))
//     }
//
// This will generate the following output:
//
//     Purging 0
//     Purging 1
//     ...
//     Purging 764
//     Purging 765
//
// Note:
//
// * The unit of item sizes is not defined; whatever it is, once the sum
// exceeds the maximum cache size, elements start getting purged until it
// drops below the threshold again.
//
// * The integers are passed by value. Caching pointers is, of course, Okay,
// but be careful when caching a memory location that holds two different
// values at different points in time; updating the value of a pointer after
// caching it will change the cached value.
//
package lrucache

type Cache struct {
	// Feel free to change this whenever. The units are not bytes but just
	// whatever unit it is that your cache entries return from Size(). If
	// (roughly) all cached items are going to be (roughly) the same size it
	// makes sense to return 1 from Size() and set MaxSize to the maximum number
	// of elements you want to allow in cache.
	MaxSize int64
	size    int64
	entries map[string]*cacheEntry
	// Cache operations are pushed down this channel to the main cache loop
	opChan           chan operation
	lruHead, lruTail *cacheEntry
	// If not nil, invoked for every cache miss.
	onMiss func(string) Cacheable
}

func (c *Cache) Size() int64 {
	return c.size
}

// Anything that implements this interface can be stored in a cache. Two
// different types can share the same cache, all that matters is that they
// implement this interface.
type Cacheable interface {
	// See Cache.MaxSize for an explanation
	Size() int64
}

type NotifyPurge interface {
	Cacheable
	// Called once when the element is purged from cache. The deleted boolean
	// indicates whether this call was the result of a call to Cache.Delete to
	// explicitly delete this item.  Possible reasons for this method to get
	// called:
	//
	// * Cache is growing too large and this is the least used item (deleted =
	// false)
	//
	// * This item was explicitly deleted using Cache.Delete(id) (deleted =
	// true)
	//
	// * A new element with the same key is stored (deleted = false)
	//
	// For most types of cached elements, this can just be a NOP. A real
	// example is a session cache where sessions are not stored in a database
	// until they are purged from the memory cache. As long as the memory cache
	// is large enough to hold all of them, they expire before the cache grows
	// too large and no database connection is ever needed. This OnPurge
	// implementation would store items to a database iff deleted == false.
	//
	// Called from within a private goroutine, but never called concurrently
	// with other elements' OnPurge().
	OnPurge(deleted bool)
}

// Requests that are passed to the cache managing goroutine
type operation interface{}

type reqSet struct {
	id      string
	payload Cacheable
}

type reqGet struct {
	id string
	// Cache goroutine pushes result down this channel (if any) and closes it
	reply chan Cacheable
}

type reqDelete string

// Used only for testing
type reqPing chan (bool)

type reqOnMissFunc func(string) Cacheable

type cacheEntry struct {
	payload Cacheable
	id      string
	// Pointers for LRU cache
	prev, next *cacheEntry
}

// Only call c.OnPurge() if c implements NotifyPurge.
func safeOnPurge(c Cacheable, deleted bool) {
	if t, ok := c.(NotifyPurge); ok {
		t.OnPurge(deleted)
	}
	return
}

func removeEntry(c *Cache, e *cacheEntry) {
	delete(c.entries, e.id)
	if e.prev == nil {
		c.lruTail = e.next
	} else {
		e.prev.next = e.next
	}
	if e.next == nil {
		c.lruHead = e.prev
	} else {
		e.next.prev = e.prev
	}
	c.size -= e.payload.Size()
	return
}

// Purge the least recently used from the cache
func purgeLRU(c *Cache) {
	safeOnPurge(c.lruTail.payload, false)
	removeEntry(c, c.lruTail)
	return
}

// Trim the cache until its size <= max size
func trimCache(c *Cache) {
	for c.size > c.MaxSize {
		purgeLRU(c)
	}
	return
}

// Not safe for use in concurrent goroutines
func directSet(c *Cache, req reqSet) {
	// Overwrite old entry
	if old, ok := c.entries[req.id]; ok {
		safeOnPurge(old.payload, false)
		removeEntry(c, old)
	}
	e := cacheEntry{payload: req.payload, id: req.id}
	if len(c.entries) == 0 {
		c.lruTail = &e
		c.lruHead = &e
		e.next = nil
		e.prev = nil
	} else {
		c.lruHead.next = &e
		e.prev = c.lruHead
		c.lruHead = &e
	}
	c.size += e.payload.Size()
	c.entries[req.id] = &e
	trimCache(c)
	return
}

// Not safe for use in concurrent goroutines
func directDelete(c *Cache, req reqDelete) {
	id := string(req)
	e, ok := c.entries[id]
	if ok {
		safeOnPurge(e.payload, true)
		removeEntry(c, e)
	}
	return
}

// Not safe for use in concurrent goroutines
func directGet(c *Cache, req reqGet) {
	e, ok := c.entries[req.id]
	if ok {
		req.reply <- e.payload
	} else {
		if c.onMiss != nil {
			p := c.onMiss(req.id)
			if p != nil {
				req.reply <- p
				close(req.reply)
				directSet(c, reqSet{req.id, p})
				return
			}
		}
	}
	close(req.reply)
	if !ok || e.next == nil {
		return
	}
	// Put element at the start of the LRU list
	if e.prev != nil {
		e.prev.next = e.next
	} else {
		c.lruTail = e.next
	}
	e.next.prev = e.prev
	e.prev = c.lruHead
	c.lruHead = e
	return
}

func (c *Cache) Init(maxsize int64) {
	c.MaxSize = maxsize
	c.opChan = make(chan operation)
	c.entries = map[string]*cacheEntry{}
	go func() {
		for op := range c.opChan {
			switch req := op.(type) {
			case reqSet:
				directSet(c, req)
			case reqDelete:
				directDelete(c, req)
			case reqGet:
				directGet(c, req)
			case reqPing:
				req <- true
			case reqOnMissFunc:
				c.onMiss = req
			default:
				panic("Illegal cache operation")
			}
		}
	}()
	return
}

// Store this item in cache. Panics if the cacheable is nil.
func (c *Cache) Set(id string, p Cacheable) {
	if p == nil {
		panic("Cacheable value must not be nil")
	}
	c.opChan <- reqSet{payload: p, id: id}
	return
}

func (c *Cache) Get(id string) (Cacheable, bool) {
	req := reqGet{id: id, reply: make(chan Cacheable)}
	c.opChan <- req
	e, ok := <-req.reply
	return e, ok
}

func (c *Cache) Delete(id string) {
	c.opChan <- reqDelete(id)
}

// Used to populate the cache if an entry is not found. If result is not nil,
// it is stored in cache and returned from Get. Call with f is nil to clear.
func (c *Cache) OnMiss(f func(string) Cacheable) {
	c.opChan <- reqOnMissFunc(f)
}

// Create and initialize a new cache, ready for use.
func New(maxsize int64) *Cache {
	var c Cache
	c.Init(maxsize)
	return &c
}

// Shared cache for configuration-less use

var sharedCache Cache

// Only necessary if you plan on using non-methods Get and Set.
func InitShared(maxsize int64) {
	sharedCache.Init(maxsize)
}

func Get(id string) (Cacheable, bool) {
	return sharedCache.Get(id)
}

func Set(id string, c Cacheable) {
	sharedCache.Set(id, c)
	return
}

func Delete(id string) {
	sharedCache.Delete(id)
	return
}
<commit_msg>Make size operations on cache safe for goroutines<commit_after>// Copyright © 2012 Hraban Luyat <hraban@0brg.net>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.

// Light-weight in-memory LRU (object) cache library for Go.
//
// To use this library, first create a cache:
//
//      c := lrucache.New(1234)
//
// Then define a type that implements the Cacheable interface:
//
//      type cacheableInt int
//
//      func (i cacheableInt) OnPurge(deleted bool) {
//          fmt.Printf("Purging %d\n", i)
//      }
//
//      func (i cacheableInt) Size() int64 {
//          return 1
//      }
//
// Finally:
//
//     for i := 0; i < 2000; i++ {
//         c.Set(strconv.Itoa(i), cacheableInt(i))
//     }
//
// This will generate the following output:
//
//     Purging 0
//     Purging 1
//     ...
//     Purging 764
//     Purging 765
//
// Note:
//
// * The unit of item sizes is not defined; whatever it is, once the sum
// exceeds the maximum cache size, elements start getting purged until it
// drops below the threshold again.
//
// * The integers are passed by value. Caching pointers is, of course, Okay,
// but be careful when caching a memory location that holds two different
// values at different points in time; updating the value of a pointer after
// caching it will change the cached value.
//
package lrucache

type Cache struct {
	maxSize int64
	size    int64
	entries map[string]*cacheEntry
	// Cache operations are pushed down this channel to the main cache loop
	opChan           chan operation
	lruHead, lruTail *cacheEntry
	// If not nil, invoked for every cache miss.
	onMiss func(string) Cacheable
}

// Anything that implements this interface can be stored in a cache. Two
// different types can share the same cache, all that matters is that they
// implement this interface.
type Cacheable interface {
	// See Cache.MaxSize() for an explanation
	Size() int64
}

type NotifyPurge interface {
	Cacheable
	// Called once when the element is purged from cache. The deleted boolean
	// indicates whether this call was the result of a call to Cache.Delete to
	// explicitly delete this item.  Possible reasons for this method to get
	// called:
	//
	// * Cache is growing too large and this is the least used item (deleted =
	// false)
	//
	// * This item was explicitly deleted using Cache.Delete(id) (deleted =
	// true)
	//
	// * A new element with the same key is stored (deleted = false)
	//
	// For most types of cached elements, this can just be a NOP. A real
	// example is a session cache where sessions are not stored in a database
	// until they are purged from the memory cache. As long as the memory cache
	// is large enough to hold all of them, they expire before the cache grows
	// too large and no database connection is ever needed. This OnPurge
	// implementation would store items to a database iff deleted == false.
	//
	// Called from within a private goroutine, but never called concurrently
	// with other elements' OnPurge().
	OnPurge(deleted bool)
}

// Requests that are passed to the cache managing goroutine
type operation interface{}

type reqSet struct {
	id      string
	payload Cacheable
}

type reqGet struct {
	id string
	// Cache goroutine pushes result down this channel (if any) and closes it
	reply chan Cacheable
}

type reqDelete string

// Used only for testing
type reqPing chan (bool)

type reqOnMissFunc func(string) Cacheable

type reqMaxSize int64

type reqGetSize chan<- int64

type cacheEntry struct {
	payload Cacheable
	id      string
	// Pointers for LRU cache
	prev, next *cacheEntry
}

// Only call c.OnPurge() if c implements NotifyPurge.
func safeOnPurge(c Cacheable, deleted bool) {
	if t, ok := c.(NotifyPurge); ok {
		t.OnPurge(deleted)
	}
	return
}

func removeEntry(c *Cache, e *cacheEntry) {
	delete(c.entries, e.id)
	if e.prev == nil {
		c.lruTail = e.next
	} else {
		e.prev.next = e.next
	}
	if e.next == nil {
		c.lruHead = e.prev
	} else {
		e.next.prev = e.prev
	}
	c.size -= e.payload.Size()
	return
}

// Purge the least recently used from the cache
func purgeLRU(c *Cache) {
	safeOnPurge(c.lruTail.payload, false)
	removeEntry(c, c.lruTail)
	return
}

// Trim the cache until its size <= max size
func trimCache(c *Cache) {
	for c.size > c.maxSize {
		purgeLRU(c)
	}
	return
}

// Not safe for use in concurrent goroutines
func directSet(c *Cache, req reqSet) {
	// Overwrite old entry
	if old, ok := c.entries[req.id]; ok {
		safeOnPurge(old.payload, false)
		removeEntry(c, old)
	}
	e := cacheEntry{payload: req.payload, id: req.id}
	if len(c.entries) == 0 {
		c.lruTail = &e
		c.lruHead = &e
		e.next = nil
		e.prev = nil
	} else {
		c.lruHead.next = &e
		e.prev = c.lruHead
		c.lruHead = &e
	}
	c.size += e.payload.Size()
	c.entries[req.id] = &e
	trimCache(c)
	return
}

// Not safe for use in concurrent goroutines
func directDelete(c *Cache, req reqDelete) {
	id := string(req)
	e, ok := c.entries[id]
	if ok {
		safeOnPurge(e.payload, true)
		removeEntry(c, e)
	}
	return
}

// Not safe for use in concurrent goroutines
func directGet(c *Cache, req reqGet) {
	e, ok := c.entries[req.id]
	if ok {
		req.reply <- e.payload
	} else {
		if c.onMiss != nil {
			p := c.onMiss(req.id)
			if p != nil {
				req.reply <- p
				close(req.reply)
				directSet(c, reqSet{req.id, p})
				return
			}
		}
	}
	close(req.reply)
	if !ok || e.next == nil {
		return
	}
	// Put element at the start of the LRU list
	if e.prev != nil {
		e.prev.next = e.next
	} else {
		c.lruTail = e.next
	}
	e.next.prev = e.prev
	e.prev = c.lruHead
	c.lruHead = e
	return
}

func (c *Cache) Init(maxsize int64) {
	c.maxSize = maxsize
	c.opChan = make(chan operation)
	c.entries = map[string]*cacheEntry{}
	go func() {
		for op := range c.opChan {
			switch req := op.(type) {
			case reqSet:
				directSet(c, req)
			case reqDelete:
				directDelete(c, req)
			case reqGet:
				directGet(c, req)
			case reqPing:
				req <- true
			case reqOnMissFunc:
				c.onMiss = req
			case reqMaxSize:
				c.maxSize = int64(req)
				trimCache(c)
			case reqGetSize:
				req <- c.size
				close(req)
			default:
				panic("Illegal cache operation")
			}
		}
	}()
	return
}

// Store this item in cache. Panics if the cacheable is nil.
func (c *Cache) Set(id string, p Cacheable) {
	if p == nil {
		panic("Cacheable value must not be nil")
	}
	c.opChan <- reqSet{payload: p, id: id}
	return
}

func (c *Cache) Get(id string) (Cacheable, bool) {
	req := reqGet{id: id, reply: make(chan Cacheable)}
	c.opChan <- req
	e, ok := <-req.reply
	return e, ok
}

func (c *Cache) Delete(id string) {
	c.opChan <- reqDelete(id)
}

// Used to populate the cache if an entry is not found. If result is not nil,
// it is stored in cache and returned from Get. Call with f is nil to clear.
func (c *Cache) OnMiss(f func(string) Cacheable) {
	c.opChan <- reqOnMissFunc(f)
}

// Feel free to change this whenever. The units are not bytes but just whatever
// unit it is that your cache entries return from Size(). If (roughly) all
// cached items are going to be (roughly) the same size it makes sense to
// return 1 from Size() and set maxSize to the maximum number of elements you
// want to allow in cache.
func (c *Cache) MaxSize(i int64) {
	c.opChan <- reqMaxSize(i)
}

func (c *Cache) Size() int64 {
	reply := make(chan int64)
	c.opChan <- reqGetSize(reply)
	return <-reply
}

// Create and initialize a new cache, ready for use.
func New(maxsize int64) *Cache {
	var c Cache
	c.Init(maxsize)
	return &c
}

// Shared cache for configuration-less use

var sharedCache Cache

// Only necessary if you plan on using non-methods Get and Set.
func InitShared(maxsize int64) {
	sharedCache.Init(maxsize)
}

func Get(id string) (Cacheable, bool) {
	return sharedCache.Get(id)
}

func Set(id string, c Cacheable) {
	sharedCache.Set(id, c)
	return
}

func Delete(id string) {
	sharedCache.Delete(id)
	return
}
<|endoftext|>