<commit_before>package docker

//
//  Types extracted from Docker
//

import (
	"time"

	"github.com/containers/image/pkg/strslice"
	"github.com/opencontainers/go-digest"
)

// github.com/docker/docker/image/rootfs.go
const TypeLayers = "layers"

// github.com/docker/docker/image/rootfs.go
// RootFS describes images root filesystem
// This is currently a placeholder that only supports layers. In the future
// this can be made into an interface that supports different implementations.
type RootFS struct {
	Type    string          `json:"type"`
	DiffIDs []digest.Digest `json:"diff_ids,omitempty"`
}

// github.com/docker/docker/image/image.go
// History stores build commands that were used to create an image
type History struct {
	// Created is the timestamp at which the image was created
	Created time.Time `json:"created"`
	// Author is the name of the author that was specified when committing the image
	Author string `json:"author,omitempty"`
	// CreatedBy keeps the Dockerfile command used while building the image
	CreatedBy string `json:"created_by,omitempty"`
	// Comment is the commit message that was set when committing the image
	Comment string `json:"comment,omitempty"`
	// EmptyLayer is set to true if this history item did not generate a
	// layer. Otherwise, the history item is associated with the next
	// layer in the RootFS section.
	EmptyLayer bool `json:"empty_layer,omitempty"`
}

// github.com/docker/docker/image/image.go
// ID is the content-addressable ID of an image.
type ID digest.Digest

// github.com/docker/docker/api/types/container/config.go
// HealthConfig holds configuration settings for the HEALTHCHECK feature.
type HealthConfig struct {
	// Test is the test to perform to check that the container is healthy.
	// An empty slice means to inherit the default.
	// The options are:
	// {} : inherit healthcheck
	// {"NONE"} : disable healthcheck
	// {"CMD", args...} : exec arguments directly
	// {"CMD-SHELL", command} : run command with system's default shell
	Test []string `json:",omitempty"`

	// Zero means to inherit. Durations are expressed as integer nanoseconds.
	Interval time.Duration `json:",omitempty"` // Interval is the time to wait between checks.
	Timeout  time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung.

	// Retries is the number of consecutive failures needed to consider a container as unhealthy.
	// Zero means inherit.
	Retries int `json:",omitempty"`
}

// github.com/docker/go-connections/nat/nat.go
// PortSet is a collection of structs indexed by Port
type PortSet map[Port]struct{}

// github.com/docker/go-connections/nat/nat.go
// Port is a string containing port number and protocol in the format "80/tcp"
type Port string

// github.com/docker/docker/api/types/container/config.go
// Config contains the configuration data about a container.
// It should hold only portable information about the container.
// Here, "portable" means "independent from the host we are running on".
// Non-portable information *should* appear in HostConfig.
// All fields added to this struct must be marked `omitempty` to keep getting
// predictable hashes from the old `v1Compatibility` configuration.
type Config struct {
	Hostname        string              // Hostname
	Domainname      string              // Domainname
	User            string              // User that will run the command(s) inside the container, also support user:group
	AttachStdin     bool                // Attach the standard input, makes possible user interaction
	AttachStdout    bool                // Attach the standard output
	AttachStderr    bool                // Attach the standard error
	ExposedPorts    PortSet             `json:",omitempty"` // List of exposed ports
	Tty             bool                // Attach standard streams to a tty, including stdin if it is not closed.
	OpenStdin       bool                // Open stdin
	StdinOnce       bool                // If true, close stdin after the 1 attached client disconnects.
	Env             []string            // List of environment variable to set in the container
	Cmd             strslice.StrSlice   // Command to run when starting the container
	Healthcheck     *HealthConfig       `json:",omitempty"` // Healthcheck describes how to check the container is healthy
	ArgsEscaped     bool                `json:",omitempty"` // True if command is already escaped (Windows specific)
	Image           string              // Name of the image as it was passed by the operator (e.g. could be symbolic)
	Volumes         map[string]struct{} // List of volumes (mounts) used for the container
	WorkingDir      string              // Current directory (PWD) in the command will be launched
	Entrypoint      strslice.StrSlice   // Entrypoint to run when starting the container
	NetworkDisabled bool                `json:",omitempty"` // Is network disabled
	MacAddress      string              `json:",omitempty"` // Mac Address of the container
	OnBuild         []string            // ONBUILD metadata that were defined on the image Dockerfile
	Labels          map[string]string   // List of labels set to this container
	StopSignal      string              `json:",omitempty"` // Signal to stop a container
	StopTimeout     *int                `json:",omitempty"` // Timeout (in seconds) to stop a container
	Shell           strslice.StrSlice   `json:",omitempty"` // Shell for shell-form of RUN, CMD, ENTRYPOINT
}

// github.com/docker/docker/image/image.go
// V1Image stores the V1 image configuration.
type V1Image struct {
	// ID is a unique 64 character identifier of the image
	ID string `json:"id,omitempty"`
	// Parent is the ID of the parent image
	Parent string `json:"parent,omitempty"`
	// Comment is the commit message that was set when committing the image
	Comment string `json:"comment,omitempty"`
	// Created is the timestamp at which the image was created
	Created time.Time `json:"created"`
	// Container is the id of the container used to commit
	Container string `json:"container,omitempty"`
	// ContainerConfig is the configuration of the container that is committed into the image
	ContainerConfig Config `json:"container_config,omitempty"`
	// DockerVersion specifies the version of Docker that was used to build the image
	DockerVersion string `json:"docker_version,omitempty"`
	// Author is the name of the author that was specified when committing the image
	Author string `json:"author,omitempty"`
	// Config is the configuration of the container received from the client
	Config *Config `json:"config,omitempty"`
	// Architecture is the hardware that the image is build and runs on
	Architecture string `json:"architecture,omitempty"`
	// OS is the operating system used to build and run the image
	OS string `json:"os,omitempty"`
	// Size is the total size of the image including all layers it is composed of
	Size int64 `json:",omitempty"`
}

// github.com/docker/docker/image/image.go
// Image stores the image configuration
type Image struct {
	V1Image
	Parent     ID        `json:"parent,omitempty"`
	RootFS     *RootFS   `json:"rootfs,omitempty"`
	History    []History `json:"history,omitempty"`
	OSVersion  string    `json:"os.version,omitempty"`
	OSFeatures []string  `json:"os.features,omitempty"`

	// rawJSON caches the immutable JSON associated with this image.
	rawJSON []byte

	// computedID is the ID computed from the hash of the image config.
	// Not to be confused with the legacy V1 ID in V1Image.
	computedID ID
}
<commit_msg>Update reference comments for docker types origins<commit_after>package docker

//
//  Types extracted from Docker
//

import (
	"time"

	"github.com/containers/image/pkg/strslice"
	"github.com/opencontainers/go-digest"
)

// github.com/moby/moby/image/rootfs.go
const TypeLayers = "layers"

// github.com/moby/moby/image/rootfs.go
// RootFS describes images root filesystem
// This is currently a placeholder that only supports layers. In the future
// this can be made into an interface that supports different implementations.
type RootFS struct {
	Type    string          `json:"type"`
	DiffIDs []digest.Digest `json:"diff_ids,omitempty"`
}

// github.com/moby/moby/image/image.go
// History stores build commands that were used to create an image
type History struct {
	// Created is the timestamp at which the image was created
	Created time.Time `json:"created"`
	// Author is the name of the author that was specified when committing the image
	Author string `json:"author,omitempty"`
	// CreatedBy keeps the Dockerfile command used while building the image
	CreatedBy string `json:"created_by,omitempty"`
	// Comment is the commit message that was set when committing the image
	Comment string `json:"comment,omitempty"`
	// EmptyLayer is set to true if this history item did not generate a
	// layer. Otherwise, the history item is associated with the next
	// layer in the RootFS section.
	EmptyLayer bool `json:"empty_layer,omitempty"`
}

// github.com/moby/moby/image/image.go
// ID is the content-addressable ID of an image.
type ID digest.Digest

// github.com/moby/moby/api/types/container/config.go
// HealthConfig holds configuration settings for the HEALTHCHECK feature.
type HealthConfig struct {
	// Test is the test to perform to check that the container is healthy.
	// An empty slice means to inherit the default.
	// The options are:
	// {} : inherit healthcheck
	// {"NONE"} : disable healthcheck
	// {"CMD", args...} : exec arguments directly
	// {"CMD-SHELL", command} : run command with system's default shell
	Test []string `json:",omitempty"`

	// Zero means to inherit. Durations are expressed as integer nanoseconds.
	Interval time.Duration `json:",omitempty"` // Interval is the time to wait between checks.
	Timeout  time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung.

	// Retries is the number of consecutive failures needed to consider a container as unhealthy.
	// Zero means inherit.
	Retries int `json:",omitempty"`
}

// github.com/docker/go-connections/nat/nat.go
// PortSet is a collection of structs indexed by Port
type PortSet map[Port]struct{}

// github.com/docker/go-connections/nat/nat.go
// Port is a string containing port number and protocol in the format "80/tcp"
type Port string

// github.com/moby/moby/api/types/container/config.go
// Config contains the configuration data about a container.
// It should hold only portable information about the container.
// Here, "portable" means "independent from the host we are running on".
// Non-portable information *should* appear in HostConfig.
// All fields added to this struct must be marked `omitempty` to keep getting
// predictable hashes from the old `v1Compatibility` configuration.
type Config struct {
	Hostname        string              // Hostname
	Domainname      string              // Domainname
	User            string              // User that will run the command(s) inside the container, also support user:group
	AttachStdin     bool                // Attach the standard input, makes possible user interaction
	AttachStdout    bool                // Attach the standard output
	AttachStderr    bool                // Attach the standard error
	ExposedPorts    PortSet             `json:",omitempty"` // List of exposed ports
	Tty             bool                // Attach standard streams to a tty, including stdin if it is not closed.
	OpenStdin       bool                // Open stdin
	StdinOnce       bool                // If true, close stdin after the 1 attached client disconnects.
	Env             []string            // List of environment variable to set in the container
	Cmd             strslice.StrSlice   // Command to run when starting the container
	Healthcheck     *HealthConfig       `json:",omitempty"` // Healthcheck describes how to check the container is healthy
	ArgsEscaped     bool                `json:",omitempty"` // True if command is already escaped (Windows specific)
	Image           string              // Name of the image as it was passed by the operator (e.g. could be symbolic)
	Volumes         map[string]struct{} // List of volumes (mounts) used for the container
	WorkingDir      string              // Current directory (PWD) in the command will be launched
	Entrypoint      strslice.StrSlice   // Entrypoint to run when starting the container
	NetworkDisabled bool                `json:",omitempty"` // Is network disabled
	MacAddress      string              `json:",omitempty"` // Mac Address of the container
	OnBuild         []string            // ONBUILD metadata that were defined on the image Dockerfile
	Labels          map[string]string   // List of labels set to this container
	StopSignal      string              `json:",omitempty"` // Signal to stop a container
	StopTimeout     *int                `json:",omitempty"` // Timeout (in seconds) to stop a container
	Shell           strslice.StrSlice   `json:",omitempty"` // Shell for shell-form of RUN, CMD, ENTRYPOINT
}

// github.com/moby/moby/image/image.go
// V1Image stores the V1 image configuration.
type V1Image struct {
	// ID is a unique 64 character identifier of the image
	ID string `json:"id,omitempty"`
	// Parent is the ID of the parent image
	Parent string `json:"parent,omitempty"`
	// Comment is the commit message that was set when committing the image
	Comment string `json:"comment,omitempty"`
	// Created is the timestamp at which the image was created
	Created time.Time `json:"created"`
	// Container is the id of the container used to commit
	Container string `json:"container,omitempty"`
	// ContainerConfig is the configuration of the container that is committed into the image
	ContainerConfig Config `json:"container_config,omitempty"`
	// DockerVersion specifies the version of Docker that was used to build the image
	DockerVersion string `json:"docker_version,omitempty"`
	// Author is the name of the author that was specified when committing the image
	Author string `json:"author,omitempty"`
	// Config is the configuration of the container received from the client
	Config *Config `json:"config,omitempty"`
	// Architecture is the hardware that the image is build and runs on
	Architecture string `json:"architecture,omitempty"`
	// OS is the operating system used to build and run the image
	OS string `json:"os,omitempty"`
	// Size is the total size of the image including all layers it is composed of
	Size int64 `json:",omitempty"`
}

// github.com/moby/moby/image/image.go
// Image stores the image configuration
type Image struct {
	V1Image
	Parent     ID        `json:"parent,omitempty"`
	RootFS     *RootFS   `json:"rootfs,omitempty"`
	History    []History `json:"history,omitempty"`
	OSVersion  string    `json:"os.version,omitempty"`
	OSFeatures []string  `json:"os.features,omitempty"`

	// rawJSON caches the immutable JSON associated with this image.
	rawJSON []byte

	// computedID is the ID computed from the hash of the image config.
	// Not to be confused with the legacy V1 ID in V1Image.
	computedID ID
}
<|endoftext|>


<commit_before>package common

import (
	"net"
	"fmt"
	"errors"
	"encoding/binary"
)

type Session struct {
	addr uint32
	table map[uint32]*net.UDPAddr
	next_addr uint8
}

func NewSessions(addr net.IP) *Session {
	return &Session {
		binary.BigEndian.Uint32(addr.To4()),
		make(map[uint32]*net.UDPAddr),
		2,
	}
}

func (n *Session) NewClient(tuple *net.UDPAddr) uint32 {
	result := n.addr & 0xffffff00 | uint32(n.next_addr)
	n.next_addr++

	n.table[result] = tuple
	return result
}

func (n *Session) Lookup(addr uint32) (*net.UDPAddr, error) {
	tuple, ok := n.table[addr]
	if !ok {
		msg := fmt.Sprintf("No session found for internal IP address: %d", addr)
		return nil, errors.New(msg)
	}
	return tuple, nil
}<commit_msg>Better logging<commit_after>package common

import (
	"net"
	"fmt"
	"errors"
	"encoding/binary"
)

type Session struct {
	addr uint32
	table map[uint32]*net.UDPAddr
	next_addr uint8
}

func NewSessions(addr net.IP) *Session {
	return &Session {
		binary.BigEndian.Uint32(addr.To4()),
		make(map[uint32]*net.UDPAddr),
		2,
	}
}

func (n *Session) NewClient(tuple *net.UDPAddr) uint32 {
	result := n.addr & 0xffffff00 | uint32(n.next_addr)
	n.next_addr++

	n.table[result] = tuple
	return result
}

func (n *Session) Lookup(addr uint32) (*net.UDPAddr, error) {
	tuple, ok := n.table[addr]
	if !ok {
		var buf net.IP
		buf.To4()
		binary.BigEndian.PutUint32(buf, addr)
		msg := fmt.Sprintf("No session found for internal IP address: %d", buf.String())
		return nil, errors.New(msg)
	}
	return tuple, nil
}<|endoftext|>


<commit_before>package workload

import (
	"errors"
	"fmt"
	"strings"

	"github.com/docker/distribution/reference"
	"github.com/rancher/norman/api/access"
	"github.com/rancher/norman/httperror"
	"github.com/rancher/norman/types"
	"github.com/rancher/norman/types/convert"
	"github.com/rancher/norman/types/values"
	projectschema "github.com/rancher/types/apis/project.cattle.io/v3/schema"
	"github.com/rancher/types/client/project/v3"
	projectclient "github.com/rancher/types/client/project/v3"
	"github.com/rancher/types/config"
	"github.com/sirupsen/logrus"
	"golang.org/x/sync/errgroup"
	corev1 "k8s.io/api/core/v1"
)

const (
	SelectorLabel = "workload.user.cattle.io/workloadselector"
)

type AggregateStore struct {
	Stores          map[string]types.Store
	Schemas         map[string]*types.Schema
	FieldToSchemaID map[string]string
}

func NewAggregateStore(schemas ...*types.Schema) *AggregateStore {
	a := &AggregateStore{
		Stores:          map[string]types.Store{},
		Schemas:         map[string]*types.Schema{},
		FieldToSchemaID: map[string]string{},
	}

	for _, schema := range schemas {
		a.Schemas[strings.ToLower(schema.ID)] = schema
		a.Stores[strings.ToLower(schema.ID)] = schema.Store
		a.FieldToSchemaID[schema.ID] = strings.ToLower(schema.ID)
	}

	return a
}

func (a *AggregateStore) Context() types.StorageContext {
	return config.UserStorageContext
}

func (a *AggregateStore) ByID(apiContext *types.APIContext, schema *types.Schema, id string) (map[string]interface{}, error) {
	store, schemaType, err := a.getStore(id)
	if err != nil {
		return nil, err
	}
	_, shortID := splitTypeAndID(id)
	return store.ByID(apiContext, a.Schemas[schemaType], shortID)
}

func (a *AggregateStore) Watch(apiContext *types.APIContext, schema *types.Schema, opt *types.QueryOptions) (chan map[string]interface{}, error) {
	readerGroup, ctx := errgroup.WithContext(apiContext.Request.Context())
	apiContext.Request = apiContext.Request.WithContext(ctx)

	events := make(chan map[string]interface{})
	for _, schema := range a.Schemas {
		streamStore(readerGroup, apiContext, schema, opt, events)
	}

	go func() {
		readerGroup.Wait()
		close(events)
	}()
	return events, nil
}

func (a *AggregateStore) List(apiContext *types.APIContext, schema *types.Schema, opt *types.QueryOptions) ([]map[string]interface{}, error) {
	items := make(chan map[string]interface{})
	g, ctx := errgroup.WithContext(apiContext.Request.Context())
	submit := func(schema *types.Schema, store types.Store) {
		g.Go(func() error {
			data, err := store.List(apiContext, schema, opt)
			if err != nil {
				return err
			}
			for _, item := range data {
				select {
				case items <- item:
				case <-ctx.Done():
					return ctx.Err()
				}
			}
			return nil
		})
	}

	for typeName, store := range a.Stores {
		submit(a.Schemas[typeName], store)
	}

	go func() {
		g.Wait()
		close(items)
	}()

	var result []map[string]interface{}
	for item := range items {
		result = append(result, item)
	}

	return result, g.Wait()
}

func (a *AggregateStore) Create(apiContext *types.APIContext, schema *types.Schema, data map[string]interface{}) (map[string]interface{}, error) {
	// deployment is default if otherwise is not specified
	kind := client.DeploymentType
	toSchema := a.Schemas[kind]
	toStore := a.Stores[kind]
	for field, schemaID := range a.FieldToSchemaID {
		if val, ok := data[field]; ok && val != nil {
			toSchema = a.Schemas[schemaID]
			toStore = a.Stores[schemaID]
			break
		}
	}

	setSelector(toSchema.ID, data)
	setWorkloadSpecificDefaults(toSchema.ID, data)
	setSecrets(apiContext, data)

	return toStore.Create(apiContext, toSchema, data)
}

func setSelector(schemaID string, data map[string]interface{}) {
	setSelector := false
	isJob := strings.EqualFold(schemaID, "job") || strings.EqualFold(schemaID, "cronJob")
	if convert.IsEmpty(data["selector"]) && !isJob {
		setSelector = true
	}
	if setSelector {
		workloadID := resolveWorkloadID(schemaID, data)
		// set selector
		data["selector"] = map[string]interface{}{
			"matchLabels": map[string]interface{}{
				SelectorLabel: workloadID,
			},
		}

		// set workload labels
		workloadLabels := convert.ToMapInterface(data["workloadLabels"])
		if workloadLabels == nil {
			workloadLabels = make(map[string]interface{})
		}
		workloadLabels[SelectorLabel] = workloadID
		data["workloadLabels"] = workloadLabels

		// set labels
		labels := convert.ToMapInterface(data["labels"])
		if labels == nil {
			labels = make(map[string]interface{})
		}
		labels[SelectorLabel] = workloadID
		data["labels"] = labels
	}
}

func setWorkloadSpecificDefaults(schemaID string, data map[string]interface{}) {
	if strings.EqualFold(schemaID, "job") || strings.EqualFold(schemaID, "cronJob") {
		// job has different defaults
		if _, ok := data["restartPolicy"]; !ok {
			logrus.Info("Setting restart policy")
			data["restartPolicy"] = "OnFailure"
		}
	}
}

func setSecrets(apiContext *types.APIContext, data map[string]interface{}) {
	if _, ok := values.GetValue(data, "imagePullSecrets"); ok {
		return
	}
	var imagePullSecrets []corev1.LocalObjectReference

	if containers, ok := values.GetSlice(data, "containers"); ok {
		imageToSecret := make(map[string]*corev1.LocalObjectReference)
		imagePullSecrets, _ = data["imagePullSecrets"].([]corev1.LocalObjectReference)

		for _, container := range containers {
			if image := convert.ToString(container["image"]); image != "" {
				if secretRef, ok := imageToSecret[image]; ok {
					imagePullSecrets = append(imagePullSecrets, *secretRef)
					continue
				}
				if name := getRepo(image); name != "" {
					if gotSecret(data["projectId"], data["namespaceId"], apiContext, name) {
						secretRef := &corev1.LocalObjectReference{}
						secretRef.Name = name
						imagePullSecrets = append(imagePullSecrets, *secretRef)
					}
				}
			}
		}
	}
	if imagePullSecrets != nil {
		values.PutValue(data, imagePullSecrets, "imagePullSecrets")
	}
}

func resolveWorkloadID(schemaID string, data map[string]interface{}) string {
	return fmt.Sprintf("%s-%s-%s", schemaID, data["namespaceId"], data["name"])
}

func (a *AggregateStore) Update(apiContext *types.APIContext, schema *types.Schema, data map[string]interface{}, id string) (map[string]interface{}, error) {
	store, schemaType, err := a.getStore(id)
	if err != nil {
		return nil, err
	}
	_, shortID := splitTypeAndID(id)
	return store.Update(apiContext, a.Schemas[schemaType], data, shortID)
}

func (a *AggregateStore) Delete(apiContext *types.APIContext, schema *types.Schema, id string) (map[string]interface{}, error) {
	store, schemaType, err := a.getStore(id)
	if err != nil {
		return nil, err
	}
	_, shortID := splitTypeAndID(id)
	return store.Delete(apiContext, a.Schemas[schemaType], shortID)
}

func (a *AggregateStore) getStore(id string) (types.Store, string, error) {
	typeName, _ := splitTypeAndID(id)
	store, ok := a.Stores[typeName]
	if !ok {
		return nil, "", httperror.NewAPIError(httperror.NotFound, "failed to find type "+typeName)
	}
	return store, typeName, nil
}

func streamStore(eg *errgroup.Group, apiContext *types.APIContext, schema *types.Schema, opt *types.QueryOptions, result chan map[string]interface{}) {
	eg.Go(func() error {
		events, err := schema.Store.Watch(apiContext, schema, opt)
		if err != nil || events == nil {
			if err != nil {
				logrus.Errorf("failed on subscribe %s: %v", schema.ID, err)
			}
			return err
		}

		logrus.Debugf("watching %s", schema.ID)

		for e := range events {
			result <- e
		}

		return errors.New("disconnect")
	})
}

func splitTypeAndID(id string) (string, string) {
	parts := strings.SplitN(id, ":", 2)
	if len(parts) < 2 {
		// Must conform
		return "", ""
	}
	return parts[0], parts[1]
}

func gotSecret(projectID interface{}, namespaceID interface{}, apiContext *types.APIContext, repoName string) bool {
	if project := convert.ToString(projectID); project != "" {
		splitID := strings.Split(project, ":")
		if len(splitID) == 2 {
			projectName := splitID[1]
			if ok := foundSecret(projectName, repoName, projectclient.SecretType, apiContext); ok {
				return true
			}
		}
	}
	if namespace := convert.ToString(namespaceID); namespace != "" {
		if ok := foundSecret(namespace, repoName, "namespacedSecret", apiContext); ok {
			return true
		}
	}
	logrus.Debugf("couldn't find secret [%s]", repoName)
	return false
}

func foundSecret(prefix string, repoName string, datatype string, apiContext *types.APIContext) bool {
	secretName := fmt.Sprintf("%s:%s", prefix, repoName)
	var secret interface{}
	if err := access.ByID(apiContext, &projectschema.Version, datatype, secretName, &secret); err == nil {
		if secretMap := convert.ToMapInterface(secret); secretMap != nil {
			if val, _ := secretMap["type"]; convert.ToString(val) == "dockerCredential" {
				return true
			}
		}
	}
	return false
}

func getRepo(image string) string {
	var repo string
	named, err := reference.ParseNormalizedNamed(image)
	if err != nil {
		logrus.Debug(err)
		return repo
	}
	return reference.Domain(named)
}
<commit_msg>imagePullSecret fix<commit_after>package workload

import (
	"errors"
	"fmt"
	"strings"

	"github.com/docker/distribution/reference"
	"github.com/rancher/norman/api/access"
	"github.com/rancher/norman/httperror"
	"github.com/rancher/norman/types"
	"github.com/rancher/norman/types/convert"
	"github.com/rancher/norman/types/values"
	projectschema "github.com/rancher/types/apis/project.cattle.io/v3/schema"
	"github.com/rancher/types/client/project/v3"
	projectclient "github.com/rancher/types/client/project/v3"
	"github.com/rancher/types/config"
	"github.com/sirupsen/logrus"
	"golang.org/x/sync/errgroup"
	corev1 "k8s.io/api/core/v1"
)

const (
	SelectorLabel = "workload.user.cattle.io/workloadselector"
)

type AggregateStore struct {
	Stores          map[string]types.Store
	Schemas         map[string]*types.Schema
	FieldToSchemaID map[string]string
}

func NewAggregateStore(schemas ...*types.Schema) *AggregateStore {
	a := &AggregateStore{
		Stores:          map[string]types.Store{},
		Schemas:         map[string]*types.Schema{},
		FieldToSchemaID: map[string]string{},
	}

	for _, schema := range schemas {
		a.Schemas[strings.ToLower(schema.ID)] = schema
		a.Stores[strings.ToLower(schema.ID)] = schema.Store
		a.FieldToSchemaID[schema.ID] = strings.ToLower(schema.ID)
	}

	return a
}

func (a *AggregateStore) Context() types.StorageContext {
	return config.UserStorageContext
}

func (a *AggregateStore) ByID(apiContext *types.APIContext, schema *types.Schema, id string) (map[string]interface{}, error) {
	store, schemaType, err := a.getStore(id)
	if err != nil {
		return nil, err
	}
	_, shortID := splitTypeAndID(id)
	return store.ByID(apiContext, a.Schemas[schemaType], shortID)
}

func (a *AggregateStore) Watch(apiContext *types.APIContext, schema *types.Schema, opt *types.QueryOptions) (chan map[string]interface{}, error) {
	readerGroup, ctx := errgroup.WithContext(apiContext.Request.Context())
	apiContext.Request = apiContext.Request.WithContext(ctx)

	events := make(chan map[string]interface{})
	for _, schema := range a.Schemas {
		streamStore(readerGroup, apiContext, schema, opt, events)
	}

	go func() {
		readerGroup.Wait()
		close(events)
	}()
	return events, nil
}

func (a *AggregateStore) List(apiContext *types.APIContext, schema *types.Schema, opt *types.QueryOptions) ([]map[string]interface{}, error) {
	items := make(chan map[string]interface{})
	g, ctx := errgroup.WithContext(apiContext.Request.Context())
	submit := func(schema *types.Schema, store types.Store) {
		g.Go(func() error {
			data, err := store.List(apiContext, schema, opt)
			if err != nil {
				return err
			}
			for _, item := range data {
				select {
				case items <- item:
				case <-ctx.Done():
					return ctx.Err()
				}
			}
			return nil
		})
	}

	for typeName, store := range a.Stores {
		submit(a.Schemas[typeName], store)
	}

	go func() {
		g.Wait()
		close(items)
	}()

	var result []map[string]interface{}
	for item := range items {
		result = append(result, item)
	}

	return result, g.Wait()
}

func (a *AggregateStore) Create(apiContext *types.APIContext, schema *types.Schema, data map[string]interface{}) (map[string]interface{}, error) {
	// deployment is default if otherwise is not specified
	kind := client.DeploymentType
	toSchema := a.Schemas[kind]
	toStore := a.Stores[kind]
	for field, schemaID := range a.FieldToSchemaID {
		if val, ok := data[field]; ok && val != nil {
			toSchema = a.Schemas[schemaID]
			toStore = a.Stores[schemaID]
			break
		}
	}

	setSelector(toSchema.ID, data)
	setWorkloadSpecificDefaults(toSchema.ID, data)
	setSecrets(apiContext, data)

	return toStore.Create(apiContext, toSchema, data)
}

func setSelector(schemaID string, data map[string]interface{}) {
	setSelector := false
	isJob := strings.EqualFold(schemaID, "job") || strings.EqualFold(schemaID, "cronJob")
	if convert.IsEmpty(data["selector"]) && !isJob {
		setSelector = true
	}
	if setSelector {
		workloadID := resolveWorkloadID(schemaID, data)
		// set selector
		data["selector"] = map[string]interface{}{
			"matchLabels": map[string]interface{}{
				SelectorLabel: workloadID,
			},
		}

		// set workload labels
		workloadLabels := convert.ToMapInterface(data["workloadLabels"])
		if workloadLabels == nil {
			workloadLabels = make(map[string]interface{})
		}
		workloadLabels[SelectorLabel] = workloadID
		data["workloadLabels"] = workloadLabels

		// set labels
		labels := convert.ToMapInterface(data["labels"])
		if labels == nil {
			labels = make(map[string]interface{})
		}
		labels[SelectorLabel] = workloadID
		data["labels"] = labels
	}
}

func setWorkloadSpecificDefaults(schemaID string, data map[string]interface{}) {
	if strings.EqualFold(schemaID, "job") || strings.EqualFold(schemaID, "cronJob") {
		// job has different defaults
		if _, ok := data["restartPolicy"]; !ok {
			logrus.Info("Setting restart policy")
			data["restartPolicy"] = "OnFailure"
		}
	}
}

func store(registries map[string]projectclient.RegistryCredential, domainToCreds map[string][]corev1.LocalObjectReference, name string) {
	for registry := range registries {
		secretRef := corev1.LocalObjectReference{Name: name}
		if _, ok := domainToCreds[registry]; ok {
			domainToCreds[registry] = append(domainToCreds[registry], secretRef)
		} else {
			domainToCreds[registry] = []corev1.LocalObjectReference{secretRef}
		}
	}
}

func getCreds(apiContext *types.APIContext) map[string][]corev1.LocalObjectReference {
	domainToCreds := make(map[string][]corev1.LocalObjectReference)
	var namespacedCreds []projectclient.NamespacedDockerCredential
	if err := access.List(apiContext, &projectschema.Version, "namespacedDockerCredential", &types.QueryOptions{}, &namespacedCreds); err == nil {
		for _, cred := range namespacedCreds {
			store(cred.Registries, domainToCreds, cred.Name)
		}
	}
	var creds []projectclient.DockerCredential
	if err := access.List(apiContext, &projectschema.Version, "dockerCredential", &types.QueryOptions{}, &creds); err == nil {
		for _, cred := range creds {
			store(cred.Registries, domainToCreds, cred.Name)
		}
	}
	return domainToCreds
}

func setSecrets(apiContext *types.APIContext, data map[string]interface{}) {
	if val, _ := values.GetValue(data, "imagePullSecrets"); val != nil {
		return
	}
	if containers, _ := values.GetSlice(data, "containers"); len(containers) > 0 {
		imagePullSecrets, _ := data["imagePullSecrets"].([]corev1.LocalObjectReference)
		domainToCreds := getCreds(apiContext)
		for _, container := range containers {
			if image := convert.ToString(container["image"]); image != "" {
				domain := getDomain(image)
				if secrets, ok := domainToCreds[domain]; ok {
					imagePullSecrets = append(imagePullSecrets, secrets...)
				}
			}
		}
		if imagePullSecrets != nil {
			values.PutValue(data, imagePullSecrets, "imagePullSecrets")
		}
	}
}

func resolveWorkloadID(schemaID string, data map[string]interface{}) string {
	return fmt.Sprintf("%s-%s-%s", schemaID, data["namespaceId"], data["name"])
}

func (a *AggregateStore) Update(apiContext *types.APIContext, schema *types.Schema, data map[string]interface{}, id string) (map[string]interface{}, error) {
	store, schemaType, err := a.getStore(id)
	if err != nil {
		return nil, err
	}
	_, shortID := splitTypeAndID(id)
	return store.Update(apiContext, a.Schemas[schemaType], data, shortID)
}

func (a *AggregateStore) Delete(apiContext *types.APIContext, schema *types.Schema, id string) (map[string]interface{}, error) {
	store, schemaType, err := a.getStore(id)
	if err != nil {
		return nil, err
	}
	_, shortID := splitTypeAndID(id)
	return store.Delete(apiContext, a.Schemas[schemaType], shortID)
}

func (a *AggregateStore) getStore(id string) (types.Store, string, error) {
	typeName, _ := splitTypeAndID(id)
	store, ok := a.Stores[typeName]
	if !ok {
		return nil, "", httperror.NewAPIError(httperror.NotFound, "failed to find type "+typeName)
	}
	return store, typeName, nil
}

func streamStore(eg *errgroup.Group, apiContext *types.APIContext, schema *types.Schema, opt *types.QueryOptions, result chan map[string]interface{}) {
	eg.Go(func() error {
		events, err := schema.Store.Watch(apiContext, schema, opt)
		if err != nil || events == nil {
			if err != nil {
				logrus.Errorf("failed on subscribe %s: %v", schema.ID, err)
			}
			return err
		}

		logrus.Debugf("watching %s", schema.ID)

		for e := range events {
			result <- e
		}

		return errors.New("disconnect")
	})
}

func splitTypeAndID(id string) (string, string) {
	parts := strings.SplitN(id, ":", 2)
	if len(parts) < 2 {
		// Must conform
		return "", ""
	}
	return parts[0], parts[1]
}

func getDomain(image string) string {
	var repo string
	named, err := reference.ParseNormalizedNamed(image)
	if err != nil {
		logrus.Debug(err)
		return repo
	}
	domain := reference.Domain(named)
	if domain == "docker.io" {
		return "index.docker.io"
	}
	return domain
}
<|endoftext|>


<commit_before>// Copyright © 2015 Matthias Benkort <matthias.benkort@gmail.com>
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package protocol

import (
	"errors"
	"time"
)

// Uplink json message format sent by the gateway
// More details can be found in the following Semtech protocol:
// https://github.com/TheThingsNetwork/packet_forwarder/blob/master/PROTOCOL.TXT
type RXPK struct {
	Chan uint      `json:"chan"` // Concentrator "IF" channel used for RX (unsigned integer)
	Codr string    `json:"codr"` // LoRa ECC coding rate identifier
	Data string    `json:"data"` // Base64 encoded RF packet payload, padded
	Datr string    `json:"-"`    // FSK datarate (unsigned in bit per second) || LoRa datarate identifier
	Freq float64   `json:"freq"` // RX Central frequency in MHx (unsigned float, Hz precision)
	Lsnr float64   `json:"lsnr"` // LoRa SNR ratio in dB (signed float, 0.1 dB precision)
	Modu string    `json:"modu"` // Modulation identifier "LORA" or "FSK"
	Rfch uint      `json:"rfch"` // Concentrator "RF chain" used for RX (unsigned integer)
	Rssi int       `json:"rssi"` // RSSI in dBm (signed integer, 1 dB precision)
	Size uint      `json:"size"` // RF packet payload size in bytes (unsigned integer)
	Stat int       `json:"stat"` // CRC status: 1 - OK, -1 = fail, 0 = no CRC
	Time time.Time `json:"-"`    // UTC time of pkt RX, us precision, ISO 8601 'compact' format
	Tmst uint      `json:"tmst"` // Internal timestamp of "RX finished" event (32b unsigned)
}

// Downlink json message format received by the gateway
// Most field are optional
// More details can be found in the following Semtech protocol:
// https://github.com/TheThingsNetwork/packet_forwarder/blob/master/PROTOCOL.TXT
type TXPK struct {
	Codr string    `json:"codr"` // LoRa ECC coding rate identifier
	Data string    `json:"data"` // Base64 encoded RF packet payload, padding optional
	Datr string    `json:"-"`    // LoRa datarate identifier (eg. SF12BW500) || FSK Datarate (unsigned, in bits per second)
	Fdev uint      `json:"fdev"` // FSK frequency deviation (unsigned integer, in Hz)
	Freq float64   `json:"freq"` // TX central frequency in MHz (unsigned float, Hz precision)
	Imme bool      `json:"imme"` // Send packet immediately (will ignore tmst & time)
	Ipol bool      `json:"ipol"` // Lora modulation polarization inversion
	Modu string    `json:"modu"` // Modulation identifier "LORA" or "FSK"
	Ncrc bool      `json:"ncrc"` // If true, disable the CRC of the physical layer (optional)
	Powe uint      `json:"powe"` // TX output power in dBm (unsigned integer, dBm precision)
	Prea uint      `json:"prea"` // RF preamble size (unsigned integer)
	Rfch uint      `json:"rfch"` // Concentrator "RF chain" used for TX (unsigned integer)
	Size uint      `json:"size"` // RF packet payload size in bytes (unsigned integer)
	Time time.Time `json:"-"`    // Send packet at a certain time (GPS synchronization required)
	Tmst uint      `json:"tmst"` // Send packet on a certain timestamp value (will ignore time)
}

// Status json message format sent by the gateway
// More details can be found in the following Semtech protocol:
// https://github.com/TheThingsNetwork/packet_forwarder/blob/master/PROTOCOL.TXT
type Stat struct {
	Ackr float64   `json:"ackr"` // Percentage of upstream datagrams that were acknowledged
	Alti int       `json:"alti"` // GPS altitude of the gateway in meter RX (integer)
	Dwnb uint      `json:"dwnb"` // Number of downlink datagrams received (unsigned integer)
	Lati float64   `json:"lati"` // GPS latitude of the gateway in degree (float, N is +)
	Long float64   `json:"long"` // GPS latitude of the gateway in dgree (float, E is +)
	Rxfw uint      `json:"rxfw"` // Number of radio packets forwarded (unsigned integer)
	Rxnb uint      `json:"rxnb"` // Number of radio packets received (unsigned integer)
	Rxok uint      `json:"rxok"` // Number of radio packets received with a valid PHY CRC
	Time time.Time `json:"-"`    // UTC 'system' time of the gateway, ISO 8601 'expanded' format
	Txnb uint      `json:"txnb"` // Number of packets emitted (unsigned integer)
}

// Packet as seen by the gateway. The payload is optional and could be nil, otherwise, it is a
// Base64 encoding of one of the related message format (RXPK, TXPK or Stat).
// - Version refers to the protocol version, always 1 here
// - Identifier refers to a packet command (PUSH_DATA, PUSH_ACK, PULL_DATA, PULL_RESP, PULL_ACK)
// - Token is a random number generated by the gateway on some request
type Packet struct {
	Version    byte
	Token      []byte
	Identifier byte
	GatewayId  []byte
	Payload    *Payload
}

type Payload struct {
	Raw  []byte  `json:"-"`
	RXPK *[]RXPK `json:"rxpk"`
	Stat *Stat   `json:"stat"`
	TXPK *TXPK   `json:"txpk"`
}

// Available packet commands
const (
	PUSH_DATA byte = iota
	PUSH_ACK
	PULL_DATA
	PULL_RESP
	PULL_ACK
)

// Parse a raw response from a server and turn in into a packet
// Will return an error if the response fields are incorrect
func Parse(raw []byte) (error, *Packet) {
	size := len(raw)

	if size < 3 {
		return errors.New("Invalid raw data format"), nil
	}

	packet := &Packet{
		raw[0],
		raw[1:3],
		raw[3],
		nil,
		nil,
	}

	if packet.Version != 0x1 {
		return errors.New("Unreckognized protocol version"), nil
	}

	if packet.Identifier > PULL_ACK {
		return errors.New("Unreckognized protocol identifier"), nil
	}

	if size >= 12 && packet.Identifier == PULL_DATA {
		packet.GatewayId = raw[4:12]
	}

	var err error
	if size > 4 && packet.Identifier == PUSH_DATA || packet.Identifier == PULL_RESP {
		err, packet.Payload = decodePayload(raw[4:])
	}

	return err, packet
}
<commit_msg>Fix missing parenthesis in conditional expression<commit_after>// Copyright © 2015 Matthias Benkort <matthias.benkort@gmail.com>
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package protocol

import (
	"errors"
	"time"
)

// Uplink json message format sent by the gateway
// More details can be found in the following Semtech protocol:
// https://github.com/TheThingsNetwork/packet_forwarder/blob/master/PROTOCOL.TXT
type RXPK struct {
	Chan uint      `json:"chan"` // Concentrator "IF" channel used for RX (unsigned integer)
	Codr string    `json:"codr"` // LoRa ECC coding rate identifier
	Data string    `json:"data"` // Base64 encoded RF packet payload, padded
	Datr string    `json:"-"`    // FSK datarate (unsigned in bit per second) || LoRa datarate identifier
	Freq float64   `json:"freq"` // RX Central frequency in MHx (unsigned float, Hz precision)
	Lsnr float64   `json:"lsnr"` // LoRa SNR ratio in dB (signed float, 0.1 dB precision)
	Modu string    `json:"modu"` // Modulation identifier "LORA" or "FSK"
	Rfch uint      `json:"rfch"` // Concentrator "RF chain" used for RX (unsigned integer)
	Rssi int       `json:"rssi"` // RSSI in dBm (signed integer, 1 dB precision)
	Size uint      `json:"size"` // RF packet payload size in bytes (unsigned integer)
	Stat int       `json:"stat"` // CRC status: 1 - OK, -1 = fail, 0 = no CRC
	Time time.Time `json:"-"`    // UTC time of pkt RX, us precision, ISO 8601 'compact' format
	Tmst uint      `json:"tmst"` // Internal timestamp of "RX finished" event (32b unsigned)
}

// Downlink json message format received by the gateway
// Most field are optional
// More details can be found in the following Semtech protocol:
// https://github.com/TheThingsNetwork/packet_forwarder/blob/master/PROTOCOL.TXT
type TXPK struct {
	Codr string    `json:"codr"` // LoRa ECC coding rate identifier
	Data string    `json:"data"` // Base64 encoded RF packet payload, padding optional
	Datr string    `json:"-"`    // LoRa datarate identifier (eg. SF12BW500) || FSK Datarate (unsigned, in bits per second)
	Fdev uint      `json:"fdev"` // FSK frequency deviation (unsigned integer, in Hz)
	Freq float64   `json:"freq"` // TX central frequency in MHz (unsigned float, Hz precision)
	Imme bool      `json:"imme"` // Send packet immediately (will ignore tmst & time)
	Ipol bool      `json:"ipol"` // Lora modulation polarization inversion
	Modu string    `json:"modu"` // Modulation identifier "LORA" or "FSK"
	Ncrc bool      `json:"ncrc"` // If true, disable the CRC of the physical layer (optional)
	Powe uint      `json:"powe"` // TX output power in dBm (unsigned integer, dBm precision)
	Prea uint      `json:"prea"` // RF preamble size (unsigned integer)
	Rfch uint      `json:"rfch"` // Concentrator "RF chain" used for TX (unsigned integer)
	Size uint      `json:"size"` // RF packet payload size in bytes (unsigned integer)
	Time time.Time `json:"-"`    // Send packet at a certain time (GPS synchronization required)
	Tmst uint      `json:"tmst"` // Send packet on a certain timestamp value (will ignore time)
}

// Status json message format sent by the gateway
// More details can be found in the following Semtech protocol:
// https://github.com/TheThingsNetwork/packet_forwarder/blob/master/PROTOCOL.TXT
type Stat struct {
	Ackr float64   `json:"ackr"` // Percentage of upstream datagrams that were acknowledged
	Alti int       `json:"alti"` // GPS altitude of the gateway in meter RX (integer)
	Dwnb uint      `json:"dwnb"` // Number of downlink datagrams received (unsigned integer)
	Lati float64   `json:"lati"` // GPS latitude of the gateway in degree (float, N is +)
	Long float64   `json:"long"` // GPS latitude of the gateway in dgree (float, E is +)
	Rxfw uint      `json:"rxfw"` // Number of radio packets forwarded (unsigned integer)
	Rxnb uint      `json:"rxnb"` // Number of radio packets received (unsigned integer)
	Rxok uint      `json:"rxok"` // Number of radio packets received with a valid PHY CRC
	Time time.Time `json:"-"`    // UTC 'system' time of the gateway, ISO 8601 'expanded' format
	Txnb uint      `json:"txnb"` // Number of packets emitted (unsigned integer)
}

// Packet as seen by the gateway. The payload is optional and could be nil, otherwise, it is a
// Base64 encoding of one of the related message format (RXPK, TXPK or Stat).
// - Version refers to the protocol version, always 1 here
// - Identifier refers to a packet command (PUSH_DATA, PUSH_ACK, PULL_DATA, PULL_RESP, PULL_ACK)
// - Token is a random number generated by the gateway on some request
type Packet struct {
	Version    byte
	Token      []byte
	Identifier byte
	GatewayId  []byte
	Payload    *Payload
}

type Payload struct {
	Raw  []byte  `json:"-"`
	RXPK *[]RXPK `json:"rxpk"`
	Stat *Stat   `json:"stat"`
	TXPK *TXPK   `json:"txpk"`
}

// Available packet commands
const (
	PUSH_DATA byte = iota
	PUSH_ACK
	PULL_DATA
	PULL_RESP
	PULL_ACK
)

// Parse a raw response from a server and turn in into a packet
// Will return an error if the response fields are incorrect
func Parse(raw []byte) (error, *Packet) {
	size := len(raw)

	if size < 3 {
		return errors.New("Invalid raw data format"), nil
	}

	packet := &Packet{
		raw[0],
		raw[1:3],
		raw[3],
		nil,
		nil,
	}

	if packet.Version != 0x1 {
		return errors.New("Unreckognized protocol version"), nil
	}

	if packet.Identifier > PULL_ACK {
		return errors.New("Unreckognized protocol identifier"), nil
	}

	if size >= 12 && packet.Identifier == PULL_DATA {
		packet.GatewayId = raw[4:12]
	}

	var err error
	if size > 4 && (packet.Identifier == PUSH_DATA || packet.Identifier == PULL_RESP) {
		err, packet.Payload = decodePayload(raw[4:])
	}

	return err, packet
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package responsewriters

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"k8s.io/apiserver/pkg/features"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apiserver/pkg/audit"
	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
	"k8s.io/apiserver/pkg/endpoints/metrics"
	"k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/apiserver/pkg/registry/rest"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	"k8s.io/apiserver/pkg/util/flushwriter"
	"k8s.io/apiserver/pkg/util/wsstream"
	utiltrace "k8s.io/utils/trace"
)

// StreamObject performs input stream negotiation from a ResourceStreamer and writes that to the response.
// If the client requests a websocket upgrade, negotiate for a websocket reader protocol (because many
// browser clients cannot easily handle binary streaming protocols).
func StreamObject(statusCode int, gv schema.GroupVersion, s runtime.NegotiatedSerializer, stream rest.ResourceStreamer, w http.ResponseWriter, req *http.Request) {
	out, flush, contentType, err := stream.InputStream(req.Context(), gv.String(), req.Header.Get("Accept"))
	if err != nil {
		ErrorNegotiated(err, s, gv, w, req)
		return
	}
	if out == nil {
		// No output provided - return StatusNoContent
		w.WriteHeader(http.StatusNoContent)
		return
	}
	defer out.Close()

	if wsstream.IsWebSocketRequest(req) {
		r := wsstream.NewReader(out, true, wsstream.NewDefaultReaderProtocols())
		if err := r.Copy(w, req); err != nil {
			utilruntime.HandleError(fmt.Errorf("error encountered while streaming results via websocket: %v", err))
		}
		return
	}

	if len(contentType) == 0 {
		contentType = "application/octet-stream"
	}
	w.Header().Set("Content-Type", contentType)
	w.WriteHeader(statusCode)
	// Flush headers, if possible
	if flusher, ok := w.(http.Flusher); ok {
		flusher.Flush()
	}
	writer := w.(io.Writer)
	if flush {
		writer = flushwriter.Wrap(w)
	}
	io.Copy(writer, out)
}

// SerializeObject renders an object in the content type negotiated by the client using the provided encoder.
// The context is optional and can be nil. This method will perform optional content compression if requested by
// a client and the feature gate for APIResponseCompression is enabled.
func SerializeObject(mediaType string, encoder runtime.Encoder, hw http.ResponseWriter, req *http.Request, statusCode int, object runtime.Object) {
	trace := utiltrace.New("SerializeObject",
		utiltrace.Field{"audit-id", request.GetAuditIDTruncated(req.Context())},
		utiltrace.Field{"method", req.Method},
		utiltrace.Field{"url", req.URL.Path},
		utiltrace.Field{"protocol", req.Proto},
		utiltrace.Field{"mediaType", mediaType},
		utiltrace.Field{"encoder", encoder.Identifier()})
	defer trace.LogIfLong(5 * time.Second)

	w := &deferredResponseWriter{
		mediaType:       mediaType,
		statusCode:      statusCode,
		contentEncoding: negotiateContentEncoding(req),
		hw:              hw,
		trace:           trace,
	}

	err := encoder.Encode(object, w)
	if err == nil {
		err = w.Close()
		if err != nil {
			// we cannot write an error to the writer anymore as the Encode call was successful.
			utilruntime.HandleError(fmt.Errorf("apiserver was unable to close cleanly the response writer: %v", err))
		}
		return
	}

	// make a best effort to write the object if a failure is detected
	utilruntime.HandleError(fmt.Errorf("apiserver was unable to write a JSON response: %v", err))
	status := ErrorToAPIStatus(err)
	candidateStatusCode := int(status.Code)
	// if the current status code is successful, allow the error's status code to overwrite it
	if statusCode >= http.StatusOK && statusCode < http.StatusBadRequest {
		w.statusCode = candidateStatusCode
	}
	output, err := runtime.Encode(encoder, status)
	if err != nil {
		w.mediaType = "text/plain"
		output = []byte(fmt.Sprintf("%s: %s", status.Reason, status.Message))
	}
	if _, err := w.Write(output); err != nil {
		utilruntime.HandleError(fmt.Errorf("apiserver was unable to write a fallback JSON response: %v", err))
	}
	w.Close()
}

var gzipPool = &sync.Pool{
	New: func() interface{} {
		gw, err := gzip.NewWriterLevel(nil, defaultGzipContentEncodingLevel)
		if err != nil {
			panic(err)
		}
		return gw
	},
}

const (
	// defaultGzipContentEncodingLevel is set to 4 which uses less CPU than the default level
	defaultGzipContentEncodingLevel = 4
	// defaultGzipThresholdBytes is compared to the size of the first write from the stream
	// (usually the entire object), and if the size is smaller no gzipping will be performed
	// if the client requests it.
	defaultGzipThresholdBytes = 128 * 1024
)

// negotiateContentEncoding returns a supported client-requested content encoding for the
// provided request. It will return the empty string if no supported content encoding was
// found or if response compression is disabled.
func negotiateContentEncoding(req *http.Request) string {
	encoding := req.Header.Get("Accept-Encoding")
	if len(encoding) == 0 {
		return ""
	}
	if !utilfeature.DefaultFeatureGate.Enabled(features.APIResponseCompression) {
		return ""
	}
	for len(encoding) > 0 {
		var token string
		if next := strings.Index(encoding, ","); next != -1 {
			token = encoding[:next]
			encoding = encoding[next+1:]
		} else {
			token = encoding
			encoding = ""
		}
		switch strings.TrimSpace(token) {
		case "gzip":
			return "gzip"
		}
	}
	return ""
}

type deferredResponseWriter struct {
	mediaType       string
	statusCode      int
	contentEncoding string

	hasWritten bool
	hw         http.ResponseWriter
	w          io.Writer

	trace *utiltrace.Trace
}

func (w *deferredResponseWriter) Write(p []byte) (n int, err error) {
	if w.trace != nil {
		// This Step usually wraps in-memory object serialization.
		w.trace.Step("About to start writing response", utiltrace.Field{"size", len(p)})

		firstWrite := !w.hasWritten
		defer func() {
			w.trace.Step("Write call finished",
				utiltrace.Field{"writer", fmt.Sprintf("%T", w.w)},
				utiltrace.Field{"size", len(p)},
				utiltrace.Field{"firstWrite", firstWrite},
				utiltrace.Field{"err", err})
		}()
	}
	if w.hasWritten {
		return w.w.Write(p)
	}
	w.hasWritten = true

	hw := w.hw
	header := hw.Header()
	switch {
	case w.contentEncoding == "gzip" && len(p) > defaultGzipThresholdBytes:
		header.Set("Content-Encoding", "gzip")
		header.Add("Vary", "Accept-Encoding")

		gw := gzipPool.Get().(*gzip.Writer)
		gw.Reset(hw)

		w.w = gw
	default:
		w.w = hw
	}

	header.Set("Content-Type", w.mediaType)
	hw.WriteHeader(w.statusCode)
	return w.w.Write(p)
}

func (w *deferredResponseWriter) Close() error {
	if !w.hasWritten {
		return nil
	}
	var err error
	switch t := w.w.(type) {
	case *gzip.Writer:
		err = t.Close()
		t.Reset(nil)
		gzipPool.Put(t)
	}
	return err
}

// WriteObjectNegotiated renders an object in the content type negotiated by the client.
func WriteObjectNegotiated(s runtime.NegotiatedSerializer, restrictions negotiation.EndpointRestrictions, gv schema.GroupVersion, w http.ResponseWriter, req *http.Request, statusCode int, object runtime.Object) {
	stream, ok := object.(rest.ResourceStreamer)
	if ok {
		requestInfo, _ := request.RequestInfoFrom(req.Context())
		metrics.RecordLongRunning(req, requestInfo, metrics.APIServerComponent, func() {
			StreamObject(statusCode, gv, s, stream, w, req)
		})
		return
	}

	_, serializer, err := negotiation.NegotiateOutputMediaType(req, s, restrictions)
	if err != nil {
		// if original statusCode was not successful we need to return the original error
		// we cannot hide it behind negotiation problems
		if statusCode < http.StatusOK || statusCode >= http.StatusBadRequest {
			WriteRawJSON(int(statusCode), object, w)
			return
		}
		status := ErrorToAPIStatus(err)
		WriteRawJSON(int(status.Code), status, w)
		return
	}

	audit.LogResponseObject(req.Context(), object, gv, s)

	encoder := s.EncoderForVersion(serializer.Serializer, gv)
	request.TrackSerializeResponseObjectLatency(req.Context(), func() {
		SerializeObject(serializer.MediaType, encoder, w, req, statusCode, object)
	})
}

// ErrorNegotiated renders an error to the response. Returns the HTTP status code of the error.
// The context is optional and may be nil.
func ErrorNegotiated(err error, s runtime.NegotiatedSerializer, gv schema.GroupVersion, w http.ResponseWriter, req *http.Request) int {
	status := ErrorToAPIStatus(err)
	code := int(status.Code)
	// when writing an error, check to see if the status indicates a retry after period
	if status.Details != nil && status.Details.RetryAfterSeconds > 0 {
		delay := strconv.Itoa(int(status.Details.RetryAfterSeconds))
		w.Header().Set("Retry-After", delay)
	}

	if code == http.StatusNoContent {
		w.WriteHeader(code)
		return code
	}

	WriteObjectNegotiated(s, negotiation.DefaultEndpointRestrictions, gv, w, req, code, status)
	return code
}

// WriteRawJSON writes a non-API object in JSON.
func WriteRawJSON(statusCode int, object interface{}, w http.ResponseWriter) {
	output, err := json.MarshalIndent(object, "", "  ")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(statusCode)
	w.Write(output)
}
<commit_msg>Reduce default gzip compression level from 4 to 1 in apiserver<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package responsewriters

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"k8s.io/apiserver/pkg/features"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apiserver/pkg/audit"
	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
	"k8s.io/apiserver/pkg/endpoints/metrics"
	"k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/apiserver/pkg/registry/rest"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	"k8s.io/apiserver/pkg/util/flushwriter"
	"k8s.io/apiserver/pkg/util/wsstream"
	utiltrace "k8s.io/utils/trace"
)

// StreamObject performs input stream negotiation from a ResourceStreamer and writes that to the response.
// If the client requests a websocket upgrade, negotiate for a websocket reader protocol (because many
// browser clients cannot easily handle binary streaming protocols).
func StreamObject(statusCode int, gv schema.GroupVersion, s runtime.NegotiatedSerializer, stream rest.ResourceStreamer, w http.ResponseWriter, req *http.Request) {
	out, flush, contentType, err := stream.InputStream(req.Context(), gv.String(), req.Header.Get("Accept"))
	if err != nil {
		ErrorNegotiated(err, s, gv, w, req)
		return
	}
	if out == nil {
		// No output provided - return StatusNoContent
		w.WriteHeader(http.StatusNoContent)
		return
	}
	defer out.Close()

	if wsstream.IsWebSocketRequest(req) {
		r := wsstream.NewReader(out, true, wsstream.NewDefaultReaderProtocols())
		if err := r.Copy(w, req); err != nil {
			utilruntime.HandleError(fmt.Errorf("error encountered while streaming results via websocket: %v", err))
		}
		return
	}

	if len(contentType) == 0 {
		contentType = "application/octet-stream"
	}
	w.Header().Set("Content-Type", contentType)
	w.WriteHeader(statusCode)
	// Flush headers, if possible
	if flusher, ok := w.(http.Flusher); ok {
		flusher.Flush()
	}
	writer := w.(io.Writer)
	if flush {
		writer = flushwriter.Wrap(w)
	}
	io.Copy(writer, out)
}

// SerializeObject renders an object in the content type negotiated by the client using the provided encoder.
// The context is optional and can be nil. This method will perform optional content compression if requested by
// a client and the feature gate for APIResponseCompression is enabled.
func SerializeObject(mediaType string, encoder runtime.Encoder, hw http.ResponseWriter, req *http.Request, statusCode int, object runtime.Object) {
	trace := utiltrace.New("SerializeObject",
		utiltrace.Field{"audit-id", request.GetAuditIDTruncated(req.Context())},
		utiltrace.Field{"method", req.Method},
		utiltrace.Field{"url", req.URL.Path},
		utiltrace.Field{"protocol", req.Proto},
		utiltrace.Field{"mediaType", mediaType},
		utiltrace.Field{"encoder", encoder.Identifier()})
	defer trace.LogIfLong(5 * time.Second)

	w := &deferredResponseWriter{
		mediaType:       mediaType,
		statusCode:      statusCode,
		contentEncoding: negotiateContentEncoding(req),
		hw:              hw,
		trace:           trace,
	}

	err := encoder.Encode(object, w)
	if err == nil {
		err = w.Close()
		if err != nil {
			// we cannot write an error to the writer anymore as the Encode call was successful.
			utilruntime.HandleError(fmt.Errorf("apiserver was unable to close cleanly the response writer: %v", err))
		}
		return
	}

	// make a best effort to write the object if a failure is detected
	utilruntime.HandleError(fmt.Errorf("apiserver was unable to write a JSON response: %v", err))
	status := ErrorToAPIStatus(err)
	candidateStatusCode := int(status.Code)
	// if the current status code is successful, allow the error's status code to overwrite it
	if statusCode >= http.StatusOK && statusCode < http.StatusBadRequest {
		w.statusCode = candidateStatusCode
	}
	output, err := runtime.Encode(encoder, status)
	if err != nil {
		w.mediaType = "text/plain"
		output = []byte(fmt.Sprintf("%s: %s", status.Reason, status.Message))
	}
	if _, err := w.Write(output); err != nil {
		utilruntime.HandleError(fmt.Errorf("apiserver was unable to write a fallback JSON response: %v", err))
	}
	w.Close()
}

var gzipPool = &sync.Pool{
	New: func() interface{} {
		gw, err := gzip.NewWriterLevel(nil, defaultGzipContentEncodingLevel)
		if err != nil {
			panic(err)
		}
		return gw
	},
}

const (
	// defaultGzipContentEncodingLevel is set to 1 which uses least CPU compared to higher levels, yet offers
	// similar compression ratios (off by at most 1.5x, but typically within 1.1x-1.3x). For further details see -
	// https://github.com/kubernetes/kubernetes/issues/112296
	defaultGzipContentEncodingLevel = 1
	// defaultGzipThresholdBytes is compared to the size of the first write from the stream
	// (usually the entire object), and if the size is smaller no gzipping will be performed
	// if the client requests it.
	defaultGzipThresholdBytes = 128 * 1024
)

// negotiateContentEncoding returns a supported client-requested content encoding for the
// provided request. It will return the empty string if no supported content encoding was
// found or if response compression is disabled.
func negotiateContentEncoding(req *http.Request) string {
	encoding := req.Header.Get("Accept-Encoding")
	if len(encoding) == 0 {
		return ""
	}
	if !utilfeature.DefaultFeatureGate.Enabled(features.APIResponseCompression) {
		return ""
	}
	for len(encoding) > 0 {
		var token string
		if next := strings.Index(encoding, ","); next != -1 {
			token = encoding[:next]
			encoding = encoding[next+1:]
		} else {
			token = encoding
			encoding = ""
		}
		switch strings.TrimSpace(token) {
		case "gzip":
			return "gzip"
		}
	}
	return ""
}

type deferredResponseWriter struct {
	mediaType       string
	statusCode      int
	contentEncoding string

	hasWritten bool
	hw         http.ResponseWriter
	w          io.Writer

	trace *utiltrace.Trace
}

func (w *deferredResponseWriter) Write(p []byte) (n int, err error) {
	if w.trace != nil {
		// This Step usually wraps in-memory object serialization.
		w.trace.Step("About to start writing response", utiltrace.Field{"size", len(p)})

		firstWrite := !w.hasWritten
		defer func() {
			w.trace.Step("Write call finished",
				utiltrace.Field{"writer", fmt.Sprintf("%T", w.w)},
				utiltrace.Field{"size", len(p)},
				utiltrace.Field{"firstWrite", firstWrite},
				utiltrace.Field{"err", err})
		}()
	}
	if w.hasWritten {
		return w.w.Write(p)
	}
	w.hasWritten = true

	hw := w.hw
	header := hw.Header()
	switch {
	case w.contentEncoding == "gzip" && len(p) > defaultGzipThresholdBytes:
		header.Set("Content-Encoding", "gzip")
		header.Add("Vary", "Accept-Encoding")

		gw := gzipPool.Get().(*gzip.Writer)
		gw.Reset(hw)

		w.w = gw
	default:
		w.w = hw
	}

	header.Set("Content-Type", w.mediaType)
	hw.WriteHeader(w.statusCode)
	return w.w.Write(p)
}

func (w *deferredResponseWriter) Close() error {
	if !w.hasWritten {
		return nil
	}
	var err error
	switch t := w.w.(type) {
	case *gzip.Writer:
		err = t.Close()
		t.Reset(nil)
		gzipPool.Put(t)
	}
	return err
}

// WriteObjectNegotiated renders an object in the content type negotiated by the client.
func WriteObjectNegotiated(s runtime.NegotiatedSerializer, restrictions negotiation.EndpointRestrictions, gv schema.GroupVersion, w http.ResponseWriter, req *http.Request, statusCode int, object runtime.Object) {
	stream, ok := object.(rest.ResourceStreamer)
	if ok {
		requestInfo, _ := request.RequestInfoFrom(req.Context())
		metrics.RecordLongRunning(req, requestInfo, metrics.APIServerComponent, func() {
			StreamObject(statusCode, gv, s, stream, w, req)
		})
		return
	}

	_, serializer, err := negotiation.NegotiateOutputMediaType(req, s, restrictions)
	if err != nil {
		// if original statusCode was not successful we need to return the original error
		// we cannot hide it behind negotiation problems
		if statusCode < http.StatusOK || statusCode >= http.StatusBadRequest {
			WriteRawJSON(int(statusCode), object, w)
			return
		}
		status := ErrorToAPIStatus(err)
		WriteRawJSON(int(status.Code), status, w)
		return
	}

	audit.LogResponseObject(req.Context(), object, gv, s)

	encoder := s.EncoderForVersion(serializer.Serializer, gv)
	request.TrackSerializeResponseObjectLatency(req.Context(), func() {
		SerializeObject(serializer.MediaType, encoder, w, req, statusCode, object)
	})
}

// ErrorNegotiated renders an error to the response. Returns the HTTP status code of the error.
// The context is optional and may be nil.
func ErrorNegotiated(err error, s runtime.NegotiatedSerializer, gv schema.GroupVersion, w http.ResponseWriter, req *http.Request) int {
	status := ErrorToAPIStatus(err)
	code := int(status.Code)
	// when writing an error, check to see if the status indicates a retry after period
	if status.Details != nil && status.Details.RetryAfterSeconds > 0 {
		delay := strconv.Itoa(int(status.Details.RetryAfterSeconds))
		w.Header().Set("Retry-After", delay)
	}

	if code == http.StatusNoContent {
		w.WriteHeader(code)
		return code
	}

	WriteObjectNegotiated(s, negotiation.DefaultEndpointRestrictions, gv, w, req, code, status)
	return code
}

// WriteRawJSON writes a non-API object in JSON.
func WriteRawJSON(statusCode int, object interface{}, w http.ResponseWriter) {
	output, err := json.MarshalIndent(object, "", "  ")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(statusCode)
	w.Write(output)
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"encoding/json"
	"github.com/Sirupsen/logrus"
	"github.com/gocraft/web"
	"io"
	"io/ioutil"
	"net/http"
	"path/filepath"
	"runtime"
	"sync/atomic"
)

// Context is a context object for gocraft/web.
type Context struct {
	body      []byte
	bodyError error

	requestID uint64

	response   web.ResponseWriter
	request    *web.Request
	HTTPStatus int

	logger     *logrus.Logger
	topologies TopologyRegistry
}

// SetLogger sets the logger to the context. Must be set before any action
// is invoked.
func (c *Context) SetLogger(l *logrus.Logger) {
	c.logger = l
}

var requestIDCounter uint64 = 0

func (c *Context) setUpContext(rw web.ResponseWriter, req *web.Request, next web.NextMiddlewareFunc) {
	c.requestID = atomic.AddUint64(&requestIDCounter, 1)
	c.response = rw
	c.request = req

	// TODO: request logging
	next(rw, req)
}

// SetTopologyRegistry sets the registry of topologies to this context. This
// method must be called in the middleware of Context.
func (c *Context) SetTopologyRegistry(r TopologyRegistry) {
	c.topologies = r
}

func (c *Context) NotFoundHandler(rw web.ResponseWriter, req *web.Request) {
	c.logger.WithFields(logrus.Fields{
		"method": req.Method,
		"uri":    req.URL.RequestURI(),
		"status": http.StatusNotFound,
	}).Error("The request URL not found")

	rw.Header().Set("Content-Type", "application/json")
	rw.WriteHeader(http.StatusNotFound)
	rw.Write([]byte(`
{
  "error": {
    "code": "E0001",
    "message": "The request URL was not found."
  }
}`))
}

// Log returns the logger having meta information.
func (c *Context) Log() *logrus.Entry {
	return c.log(1)
}

// ErrLog returns the logger with error information.
func (c *Context) ErrLog(err error) *logrus.Entry {
	return c.log(1).WithField("err", err)
}

func (c *Context) log(depth int) *logrus.Entry {
	// TODO: This is a temporary solution until logrus support filename and line number
	_, file, line, ok := runtime.Caller(depth + 1)
	if !ok {
		return c.logger.WithField("reqid", c.requestID)
	}
	file = filepath.Base(file) // only the filename at the moment
	return c.logger.WithFields(logrus.Fields{
		"file":  file,
		"line":  line,
		"reqid": c.requestID,
	})
}

func (c *Context) extractOptionStringFromPath(key string, target *string) error {
	s, ok := c.request.PathParams[key]
	if !ok {
		return nil
	}

	*target = s
	return nil
}

func (c *Context) renderJSON(status int, v interface{}) {
	c.HTTPStatus = status

	data, err := json.Marshal(v)
	if err != nil {
		// TODO logging
		c.response.Header().Set("Content-Type", "application/json")
		c.response.WriteHeader(http.StatusInternalServerError)
		c.response.Write([]byte(`
{
  "errors": [
    {
      "code": "E0006",
      "message": "internal server error"
    }
  ]
}
`))
		return
	}

	c.response.Header().Set("Content-Type", "application/json")
	c.response.WriteHeader(status)
	_, err = c.response.Write(data)
	if err != nil {
		// logging
	}
}

func (c *Context) RenderJSON(v interface{}) {
	c.renderJSON(http.StatusOK, v)
}

func (c *Context) RenderErrorJSON(e *Error) {
	e.SetRequestID(c.requestID)
	c.renderJSON(e.Status, map[string]interface{}{
		"error": e,
	})
}

// Body returns a slice containing whole request body.
// It caches the result so controllers can call this
// method as many time as it wants.
//
// When the request body is empty (i.e. Read(req.Body)
// returns io.EOF), this method returns and caches
// an empty body slice and a nil error.
func (c *Context) Body() ([]byte, error) {
	if c.body != nil || c.bodyError != nil {
		return c.body, c.bodyError
	}

	body, err := ioutil.ReadAll(c.request.Body)
	if err != nil {
		if err == io.EOF {
			// when body is empty, this method caches
			// an empty slice and a nil error.
			body = []byte{}
			err = nil
		}
	}

	// Close and replace with new ReadCloser for parsing
	// mime/multipart request body by Request.FormFile method.
	c.request.Body.Close()
	c.request.Body = ioutil.NopCloser(bytes.NewReader(body))
	c.body = body
	c.bodyError = err
	return body, err
}

// ContextGlobalVariables has fields which are shared through all contexts
// allocated for each request.
type ContextGlobalVariables struct {
	// Logger is used to write log messages.
	Logger *logrus.Logger

	// Topologies is a registry which manages topologies to support multi
	// tenancy.
	Topologies TopologyRegistry
}

// SetUpRouter creates a root router of the API server.
func SetUpRouter(prefix string, gvars ContextGlobalVariables) *web.Router {
	root := web.NewWithPrefix(Context{}, prefix)
	root.NotFound((*Context).NotFoundHandler)
	root.Middleware(func(c *Context, rw web.ResponseWriter, req *web.Request, next web.NextMiddlewareFunc) {
		c.SetLogger(gvars.Logger)
		c.topologies = gvars.Topologies
		next(rw, req)
	})
	root.Middleware((*Context).setUpContext)
	return root
}
<commit_msg>Add access logging<commit_after>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/gocraft/web"
	"io"
	"io/ioutil"
	"net/http"
	"path/filepath"
	"runtime"
	"sync/atomic"
	"time"
)

// Context is a context object for gocraft/web.
type Context struct {
	body      []byte
	bodyError error

	requestID uint64

	response   web.ResponseWriter
	request    *web.Request
	HTTPStatus int

	logger     *logrus.Logger
	topologies TopologyRegistry
}

// SetLogger sets the logger to the context. Must be set before any action
// is invoked.
func (c *Context) SetLogger(l *logrus.Logger) {
	c.logger = l
}

var requestIDCounter uint64 = 0

func (c *Context) setUpContext(rw web.ResponseWriter, req *web.Request, next web.NextMiddlewareFunc) {
	c.requestID = atomic.AddUint64(&requestIDCounter, 1)
	c.response = rw
	c.request = req

	start := time.Now()
	defer func() {
		elapsed := time.Now().Sub(start)
		// Use custom logging because file and line aren't necessary here.
		c.logger.WithFields(logrus.Fields{
			"reqid":   c.requestID,
			"reqtime": fmt.Sprintf("%d.%03d", int(elapsed/time.Second), int(elapsed%time.Second/time.Millisecond)),
			"method":  req.Method,
			"uri":     req.URL.RequestURI(),
			"status":  c.HTTPStatus,
		}).Info("Access")
	}()
	next(rw, req)
}

// SetTopologyRegistry sets the registry of topologies to this context. This
// method must be called in the middleware of Context.
func (c *Context) SetTopologyRegistry(r TopologyRegistry) {
	c.topologies = r
}

func (c *Context) NotFoundHandler(rw web.ResponseWriter, req *web.Request) {
	c.logger.WithFields(logrus.Fields{
		"method": req.Method,
		"uri":    req.URL.RequestURI(),
		"status": http.StatusNotFound,
	}).Error("The request URL not found")

	rw.Header().Set("Content-Type", "application/json")
	rw.WriteHeader(http.StatusNotFound)
	rw.Write([]byte(`
{
  "error": {
    "code": "E0001",
    "message": "The request URL was not found."
  }
}`))
}

// Log returns the logger having meta information.
func (c *Context) Log() *logrus.Entry {
	return c.log(1)
}

// ErrLog returns the logger with error information.
func (c *Context) ErrLog(err error) *logrus.Entry {
	return c.log(1).WithField("err", err)
}

func (c *Context) log(depth int) *logrus.Entry {
	// TODO: This is a temporary solution until logrus support filename and line number
	_, file, line, ok := runtime.Caller(depth + 1)
	if !ok {
		return c.logger.WithField("reqid", c.requestID)
	}
	file = filepath.Base(file) // only the filename at the moment
	return c.logger.WithFields(logrus.Fields{
		"file":  file,
		"line":  line,
		"reqid": c.requestID,
	})
}

func (c *Context) extractOptionStringFromPath(key string, target *string) error {
	s, ok := c.request.PathParams[key]
	if !ok {
		return nil
	}

	*target = s
	return nil
}

func (c *Context) renderJSON(status int, v interface{}) {
	c.HTTPStatus = status

	data, err := json.Marshal(v)
	if err != nil {
		// TODO logging
		c.response.Header().Set("Content-Type", "application/json")
		c.response.WriteHeader(http.StatusInternalServerError)
		c.response.Write([]byte(`
{
  "errors": [
    {
      "code": "E0006",
      "message": "internal server error"
    }
  ]
}
`))
		return
	}

	c.response.Header().Set("Content-Type", "application/json")
	c.response.WriteHeader(status)
	_, err = c.response.Write(data)
	if err != nil {
		// logging
	}
}

func (c *Context) RenderJSON(v interface{}) {
	c.renderJSON(http.StatusOK, v)
}

func (c *Context) RenderErrorJSON(e *Error) {
	e.SetRequestID(c.requestID)
	c.renderJSON(e.Status, map[string]interface{}{
		"error": e,
	})
}

// Body returns a slice containing whole request body.
// It caches the result so controllers can call this
// method as many time as it wants.
//
// When the request body is empty (i.e. Read(req.Body)
// returns io.EOF), this method returns and caches
// an empty body slice and a nil error.
func (c *Context) Body() ([]byte, error) {
	if c.body != nil || c.bodyError != nil {
		return c.body, c.bodyError
	}

	body, err := ioutil.ReadAll(c.request.Body)
	if err != nil {
		if err == io.EOF {
			// when body is empty, this method caches
			// an empty slice and a nil error.
			body = []byte{}
			err = nil
		}
	}

	// Close and replace with new ReadCloser for parsing
	// mime/multipart request body by Request.FormFile method.
	c.request.Body.Close()
	c.request.Body = ioutil.NopCloser(bytes.NewReader(body))
	c.body = body
	c.bodyError = err
	return body, err
}

// ContextGlobalVariables has fields which are shared through all contexts
// allocated for each request.
type ContextGlobalVariables struct {
	// Logger is used to write log messages.
	Logger *logrus.Logger

	// Topologies is a registry which manages topologies to support multi
	// tenancy.
	Topologies TopologyRegistry
}

// SetUpRouter creates a root router of the API server.
func SetUpRouter(prefix string, gvars ContextGlobalVariables) *web.Router {
	root := web.NewWithPrefix(Context{}, prefix)
	root.NotFound((*Context).NotFoundHandler)
	root.Middleware(func(c *Context, rw web.ResponseWriter, req *web.Request, next web.NextMiddlewareFunc) {
		c.SetLogger(gvars.Logger)
		c.topologies = gvars.Topologies
		next(rw, req)
	})
	root.Middleware((*Context).setUpContext)
	return root
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package prometheus

import (
	"github.com/prometheus/client_golang/prometheus"
	"k8s.io/client-go/util/workqueue"
)

// Package prometheus sets the workqueue DefaultMetricsFactory to produce
// prometheus metrics. To use this package, you just have to import it.

func init() {
	workqueue.SetProvider(prometheusMetricsProvider{})
}

type prometheusMetricsProvider struct{}

func (_ prometheusMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric {
	depth := prometheus.NewGauge(prometheus.GaugeOpts{
		Subsystem: name,
		Name:      "depth",
		Help:      "Current depth of workqueue: " + name,
	})
	prometheus.Register(depth)
	return depth
}

func (_ prometheusMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric {
	adds := prometheus.NewCounter(prometheus.CounterOpts{
		Subsystem: name,
		Name:      "adds",
		Help:      "Total number of adds handled by workqueue: " + name,
	})
	prometheus.Register(adds)
	return adds
}

func (_ prometheusMetricsProvider) NewLatencyMetric(name string) workqueue.SummaryMetric {
	latency := prometheus.NewSummary(prometheus.SummaryOpts{
		Subsystem: name,
		Name:      "queue_latency",
		Help:      "How long an item stays in workqueue" + name + " before being requested.",
	})
	prometheus.Register(latency)
	return latency
}

func (_ prometheusMetricsProvider) NewWorkDurationMetric(name string) workqueue.SummaryMetric {
	workDuration := prometheus.NewSummary(prometheus.SummaryOpts{
		Subsystem: name,
		Name:      "work_duration",
		Help:      "How long processing an item from workqueue" + name + " takes.",
	})
	prometheus.Register(workDuration)
	return workDuration
}

func (_ prometheusMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric {
	retries := prometheus.NewCounter(prometheus.CounterOpts{
		Subsystem: name,
		Name:      "retries",
		Help:      "Total number of retries handled by workqueue: " + name,
	})
	prometheus.Register(retries)
	return retries
}
<commit_msg>Implement lacked prometheus MetricsProvider methods<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package prometheus

import (
	"github.com/prometheus/client_golang/prometheus"
	"k8s.io/client-go/util/workqueue"
)

// Package prometheus sets the workqueue DefaultMetricsFactory to produce
// prometheus metrics. To use this package, you just have to import it.

func init() {
	workqueue.SetProvider(prometheusMetricsProvider{})
}

type prometheusMetricsProvider struct{}

func (_ prometheusMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric {
	depth := prometheus.NewGauge(prometheus.GaugeOpts{
		Subsystem: name,
		Name:      "depth",
		Help:      "Current depth of workqueue: " + name,
	})
	prometheus.Register(depth)
	return depth
}

func (_ prometheusMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric {
	adds := prometheus.NewCounter(prometheus.CounterOpts{
		Subsystem: name,
		Name:      "adds",
		Help:      "Total number of adds handled by workqueue: " + name,
	})
	prometheus.Register(adds)
	return adds
}

func (_ prometheusMetricsProvider) NewLatencyMetric(name string) workqueue.SummaryMetric {
	latency := prometheus.NewSummary(prometheus.SummaryOpts{
		Subsystem: name,
		Name:      "queue_latency",
		Help:      "How long an item stays in workqueue" + name + " before being requested.",
	})
	prometheus.Register(latency)
	return latency
}

func (_ prometheusMetricsProvider) NewWorkDurationMetric(name string) workqueue.SummaryMetric {
	workDuration := prometheus.NewSummary(prometheus.SummaryOpts{
		Subsystem: name,
		Name:      "work_duration",
		Help:      "How long processing an item from workqueue" + name + " takes.",
	})
	prometheus.Register(workDuration)
	return workDuration
}

func (_ prometheusMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric {
	retries := prometheus.NewCounter(prometheus.CounterOpts{
		Subsystem: name,
		Name:      "retries",
		Help:      "Total number of retries handled by workqueue: " + name,
	})
	prometheus.Register(retries)
	return retries
}

func (_ prometheusMetricsProvider) NewLongestRunningProcessorMicrosecondsMetric(name string) workqueue.SettableGaugeMetric {
	longestRunningProcessor := prometheus.NewGauge(prometheus.GaugeOpts{
		Subsystem: name,
		Name:      "longest_running_proceessror",
		Help:      "The longest running processor time",
	})
	prometheus.Register(longestRunningProcessor)
	return longestRunningProcessor
}

func (_ prometheusMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric {
	unfinishedWork := prometheus.NewGauge(prometheus.GaugeOpts{
		Subsystem: name,
		Name:      "unfinished_work",
		Help:      "The unfinished work duration",
	})
	prometheus.Register(unfinishedWork)
	return unfinishedWork
}
<|endoftext|>