<commit_before>package ploop

import "os/exec"
import "sync"

// #cgo CFLAGS: -D_GNU_SOURCE
// #cgo LDFLAGS: -lploop -lxml2 -lrt
// #include <ploop/libploop.h>
import "C"

// Possible SetVerboseLevel arguments
const (
	NoConsole  = C.PLOOP_LOG_NOCONSOLE
	NoStdout   = C.PLOOP_LOG_NOSTDOUT
	Timestamps = C.PLOOP_LOG_TIMESTAMPS
)

// SetVerboseLevel sets a level of verbosity when logging to stdout/stderr
func SetVerboseLevel(v int) {
	C.ploop_set_verbose_level(C.int(v))
}

// SetLogFile enables logging to a file and sets log file name
func SetLogFile(file string) error {
	cfile := C.CString(file)
	defer cfree(cfile)

	ret := C.ploop_set_log_file(cfile)

	return mkerr(ret)
}

// SetLogLevel sets a level of verbosity when logging to a file
func SetLogLevel(v int) {
	C.ploop_set_log_level(C.int(v))
}

// Ploop is a type containing DiskDescriptor.xml opened by the library
type Ploop struct {
	d *C.struct_ploop_disk_images_data
}

var once sync.Once

// load ploop modules
func load_kmod() {
	// try to load ploop modules
	modules := []string{"ploop", "pfmt_ploop1", "pfmt_raw", "pio_direct", "pio_nfs", "pio_kaio"}
	for _, m := range modules {
		exec.Command("modprobe", m).Run()
	}
}

// Open opens a ploop DiskDescriptor.xml, most ploop operations require it
func Open(file string) (Ploop, error) {
	var d Ploop

	once.Do(load_kmod)

	cfile := C.CString(file)
	defer cfree(cfile)

	ret := C.ploop_open_dd(&d.d, cfile)

	return d, mkerr(ret)
}

// Close closes a ploop disk descriptor when it is no longer needed
func (d Ploop) Close() {
	C.ploop_close_dd(d.d)
}

type ImageMode int

// Possible values for ImageMode
const (
	Expanded     ImageMode = C.PLOOP_EXPANDED_MODE
	Preallocated ImageMode = C.PLOOP_EXPANDED_PREALLOCATED_MODE
	Raw          ImageMode = C.PLOOP_RAW_MODE
)

// CreateParam is a set of parameters for a newly created ploop
type CreateParam struct {
	Size uint64 // image size, in kilobytes (FS size is about 10% smaller)
	Mode ImageMode
	File string // path to and a file name for base delta image
	CLog uint   // cluster block size log (6 to 15, default 11)
}

// Create creates a ploop image and its DiskDescriptor.xml
func Create(p *CreateParam) error {
	var a C.struct_ploop_create_param

	once.Do(load_kmod)

	// default image file name
	if p.File == "" {
		p.File = "root.hdd"
	}

	a.size = convertSize(p.Size)
	a.mode = C.int(p.Mode)
	if p.CLog != 0 {
		// ploop cluster block size, in 512-byte sectors
		// default is 1M cluster block size (CLog=11)
		// 2^11 = 2048 sectors, 2048*512 = 1M
		a.blocksize = 1 << p.CLog
	}
	a.image = C.CString(p.File)
	defer cfree(a.image)
	a.fstype = C.CString("ext4")
	defer cfree(a.fstype)

	ret := C.ploop_create_image(&a)
	return mkerr(ret)
}

// MountParam is a set of parameters to pass to Mount()
type MountParam struct {
	UUID     string // snapshot uuid (empty for top delta)
	Target   string // mount point (empty if no mount is needed)
	Flags    int    // bit mount flags such as MS_NOATIME
	Data     string // auxiliary mount options
	Readonly bool   // mount read-only
	Fsck     bool   // do fsck before mounting inner FS
	Quota    bool   // enable quota for inner FS
}

// Mount creates a ploop device and (optionally) mounts it
func (d Ploop) Mount(p *MountParam) (string, error) {
	var a C.struct_ploop_mount_param
	var device string

	if p.UUID != "" {
		a.guid = C.CString(p.UUID)
		defer cfree(a.guid)
	}
	if p.Target != "" {
		a.target = C.CString(p.Target)
		defer cfree(a.target)
	}

	// mount_data should not be NULL
	a.mount_data = C.CString(p.Data)
	defer cfree(a.mount_data)

	a.flags = C.int(p.Flags)
	a.ro = bool2cint(p.Readonly)
	a.fsck = bool2cint(p.Fsck)
	a.quota = bool2cint(p.Quota)

	ret := C.ploop_mount_image(d.d, &a)
	if ret == 0 {
		device = C.GoString(&a.device[0])
		// TODO? fsck_code = C.GoString(a.fsck_rc)
	}
	return device, mkerr(ret)
}

// Umount unmounts the ploop filesystem and dismantles the device
func (d Ploop) Umount() error {
	ret := C.ploop_umount_image(d.d)

	return mkerr(ret)
}

// Resize changes the ploop size. Online resize is recommended.
func (d Ploop) Resize(size uint64, offline bool) error {
	var p C.struct_ploop_resize_param

	p.size = convertSize(size)
	p.offline_resize = bool2cint(offline)

	ret := C.ploop_resize_image(d.d, &p)
	return mkerr(ret)
}

// Snapshot creates a ploop snapshot, returning its uuid
func (d Ploop) Snapshot() (string, error) {
	var p C.struct_ploop_snapshot_param
	var uuid, err = UUID()
	if err != nil {
		return "", err
	}
	p.guid = C.CString(uuid)
	defer cfree(p.guid)

	ret := C.ploop_create_snapshot(d.d, &p)
	if ret == 0 {
		uuid = C.GoString(p.guid)
	}

	return uuid, mkerr(ret)
}

// SwitchSnapshot switches to a specified snapshot,
// creates a new empty delta on top of it, and makes it a top one
// (i.e. the one new data will be written to).
// Old top delta (i.e. data modified since the last snapshot) is lost.
func (d Ploop) SwitchSnapshot(uuid string) error {
	var p C.struct_ploop_snapshot_switch_param

	p.guid = C.CString(uuid)
	defer cfree(p.guid)

	ret := C.ploop_switch_snapshot_ex(d.d, &p)

	return mkerr(ret)
}

// Possible values for SwitchSnapshotExtended flags argument
type SwitchFlag uint

const (
	// SkipDestroy, if set, modifies the behavior of
	// SwitchSnapshotExtended to not delete the old top delta, but
	// make it a snapshot and return its uuid. Without this flag,
	// old top delta (i.e. data modified since the last snapshot)
	// is lost.
	SkipDestroy SwitchFlag = C.PLOOP_SNAP_SKIP_TOPDELTA_DESTROY
	// SkipCreate flag, if set, modifies the behavior of
	// SwitchSnapshotExtended to not create a new top delta,
	// but rather transform the specified snapshot itself to be
	// the new top delta), so all new changes will be written
	// right to it. Snapshot UUID is lost in this case.
	SkipCreate SwitchFlag = C.PLOOP_SNAP_SKIP_TOPDELTA_CREATE
)

// SwitchSnapshotExtended is same as SwitchSnapshot but with additional
// flags modifying its behavior. Please see individual flags description.
// Returns uuid of what was the old top delta if SkipDestroy flag is set.
func (d Ploop) SwitchSnapshotExtended(uuid string, flags SwitchFlag) (string, error) {
	var p C.struct_ploop_snapshot_switch_param
	old_uuid := ""

	p.guid = C.CString(uuid)
	defer cfree(p.guid)

	p.flags = C.int(flags)

	if flags&SkipDestroy != 0 {
		old_uuid, err := UUID()
		if err != nil {
			return "", err
		}
		p.guid_old = C.CString(old_uuid)
		defer cfree(p.guid_old)
	}

	ret := C.ploop_switch_snapshot_ex(d.d, &p)

	return old_uuid, mkerr(ret)
}

// DeleteSnapshot deletes a snapshot (merging it down if necessary)
func (d Ploop) DeleteSnapshot(uuid string) error {
	cuuid := C.CString(uuid)
	defer cfree(cuuid)

	ret := C.ploop_delete_snapshot(d.d, cuuid)

	return mkerr(ret)
}

type ReplaceFlag int

// Possible values for ReplaceParam.flags
const (
	// KeepName renames the new file to old file name after replace;
	// note that if this option is used the old file is removed.
	KeepName ReplaceFlag = C.PLOOP_REPLACE_KEEP_NAME
)

// ReplaceParam is a set of parameters to Replace()
type ReplaceParam struct {
	File string // new image file name
	// Image to be replaced is specified by either
	// uuid, current file name, or level,
	// in the above order of preference.
	UUID    string
	CurFile string
	Level   int
	Flags   ReplaceFlag
}

// Replace replaces a ploop image to a different (but identical) one
func (d Ploop) Replace(p *ReplaceParam) error {
	var a C.struct_ploop_replace_param

	a.file = C.CString(p.File)
	defer cfree(a.file)

	if p.UUID != "" {
		a.guid = C.CString(p.UUID)
		defer cfree(a.guid)
	} else if p.CurFile != "" {
		a.cur_file = C.CString(p.CurFile)
		defer cfree(a.cur_file)
	} else {
		a.level = C.int(p.Level)
	}

	a.flags = C.int(p.Flags)

	ret := C.ploop_replace_image(d.d, &a)

	return mkerr(ret)
}

// IsMounted returns true if ploop is mounted
func (d Ploop) IsMounted() (bool, error) {
	ret := C.ploop_is_mounted(d.d)
	if ret == 0 {
		return false, nil
	} else if ret == 1 {
		return true, nil
	} else {
		// error, but no code, make our own
		return false, mkerr(E_SYS)
	}
}

// FSInfoData holds information about ploop inner file system
type FSInfoData struct {
	Blocksize   uint64
	Blocks      uint64
	Blocks_free uint64
	Inodes      uint64
	Inodes_free uint64
}

// FSInfo gets info of ploop's inner file system
func FSInfo(file string) (FSInfoData, error) {
	var cinfo C.struct_ploop_info
	var info FSInfoData
	cfile := C.CString(file)
	defer cfree(cfile)

	once.Do(load_kmod)

	ret := C.ploop_get_info_by_descr(cfile, &cinfo)
	if ret == 0 {
		info.Blocksize = uint64(cinfo.fs_bsize)
		info.Blocks = uint64(cinfo.fs_blocks)
		info.Blocks_free = uint64(cinfo.fs_bfree)
		info.Inodes = uint64(cinfo.fs_inodes)
		info.Inodes_free = uint64(cinfo.fs_ifree)
	}

	return info, mkerr(ret)
}

// ImageInfoData holds information about ploop image
type ImageInfoData struct {
	Blocks    uint64
	Blocksize uint32
	Version   int
}

// ImageInfo gets information about a ploop image
func (d Ploop) ImageInfo() (ImageInfoData, error) {
	var cinfo C.struct_ploop_spec
	var info ImageInfoData

	ret := C.ploop_get_spec(d.d, &cinfo)
	if ret == 0 {
		info.Blocks = uint64(cinfo.size)
		info.Blocksize = uint32(cinfo.blocksize)
		info.Version = int(cinfo.fmt_version)
	}

	return info, mkerr(ret)
}

// TopDeltaFile returns file name of top delta
func (d Ploop) TopDeltaFile() (string, error) {
	const len = 4096 // PATH_MAX
	var out [len]C.char

	ret := C.ploop_get_top_delta_fname(d.d, &out[0], len)
	if ret != 0 {
		// error, but no code, make our own
		return "", mkerr(E_SYS)
	}

	file := C.GoString(&out[0])
	return file, nil
}

// UUID generates a ploop UUID
func UUID() (string, error) {
	var cuuid [39]C.char

	ret := C.ploop_uuid_generate(&cuuid[0], 39)
	if ret != 0 {
		return "", mkerr(ret)
	}

	uuid := C.GoString(&cuuid[0])
	return uuid, nil
}
<commit_msg>Implement StringToImageMode() and ImageModeString()<commit_after>package ploop

import "strings"
import "os/exec"
import "sync"

// #cgo CFLAGS: -D_GNU_SOURCE
// #cgo LDFLAGS: -lploop -lxml2 -lrt
// #include <ploop/libploop.h>
import "C"

// Possible SetVerboseLevel arguments
const (
	NoConsole  = C.PLOOP_LOG_NOCONSOLE
	NoStdout   = C.PLOOP_LOG_NOSTDOUT
	Timestamps = C.PLOOP_LOG_TIMESTAMPS
)

// SetVerboseLevel sets a level of verbosity when logging to stdout/stderr
func SetVerboseLevel(v int) {
	C.ploop_set_verbose_level(C.int(v))
}

// SetLogFile enables logging to a file and sets log file name
func SetLogFile(file string) error {
	cfile := C.CString(file)
	defer cfree(cfile)

	ret := C.ploop_set_log_file(cfile)

	return mkerr(ret)
}

// SetLogLevel sets a level of verbosity when logging to a file
func SetLogLevel(v int) {
	C.ploop_set_log_level(C.int(v))
}

// Ploop is a type containing DiskDescriptor.xml opened by the library
type Ploop struct {
	d *C.struct_ploop_disk_images_data
}

var once sync.Once

// load ploop modules
func load_kmod() {
	// try to load ploop modules
	modules := []string{"ploop", "pfmt_ploop1", "pfmt_raw", "pio_direct", "pio_nfs", "pio_kaio"}
	for _, m := range modules {
		exec.Command("modprobe", m).Run()
	}
}

// Open opens a ploop DiskDescriptor.xml, most ploop operations require it
func Open(file string) (Ploop, error) {
	var d Ploop

	once.Do(load_kmod)

	cfile := C.CString(file)
	defer cfree(cfile)

	ret := C.ploop_open_dd(&d.d, cfile)

	return d, mkerr(ret)
}

// Close closes a ploop disk descriptor when it is no longer needed
func (d Ploop) Close() {
	C.ploop_close_dd(d.d)
}

type ImageMode int

// Possible values for ImageMode
const (
	Expanded     ImageMode = C.PLOOP_EXPANDED_MODE
	Preallocated ImageMode = C.PLOOP_EXPANDED_PREALLOCATED_MODE
	Raw          ImageMode = C.PLOOP_RAW_MODE
)

// StringToImageMode converts a string to ImageMode value
func StringToImageMode(s string) (ImageMode, error) {
	switch strings.ToLower(s) {
	case "expanded":
		return Expanded, nil
	case "preallocated":
		return Preallocated, nil
	case "raw":
		return Raw, nil
	default:
		return Expanded, mkerr(E_PARAM)
	}
}

// ImageModeString converts an ImageMode value to string
func ImageModeString(m ImageMode) string {
	switch m {
	case Expanded:
		return "Expanded"
	case Preallocated:
		return "Preallocated"
	case Raw:
		return "Raw"
	}
	return "<unknown>"
}

// CreateParam is a set of parameters for a newly created ploop
type CreateParam struct {
	Size uint64 // image size, in kilobytes (FS size is about 10% smaller)
	Mode ImageMode
	File string // path to and a file name for base delta image
	CLog uint   // cluster block size log (6 to 15, default 11)
}

// Create creates a ploop image and its DiskDescriptor.xml
func Create(p *CreateParam) error {
	var a C.struct_ploop_create_param

	once.Do(load_kmod)

	// default image file name
	if p.File == "" {
		p.File = "root.hdd"
	}

	a.size = convertSize(p.Size)
	a.mode = C.int(p.Mode)
	if p.CLog != 0 {
		// ploop cluster block size, in 512-byte sectors
		// default is 1M cluster block size (CLog=11)
		// 2^11 = 2048 sectors, 2048*512 = 1M
		a.blocksize = 1 << p.CLog
	}
	a.image = C.CString(p.File)
	defer cfree(a.image)
	a.fstype = C.CString("ext4")
	defer cfree(a.fstype)

	ret := C.ploop_create_image(&a)
	return mkerr(ret)
}

// MountParam is a set of parameters to pass to Mount()
type MountParam struct {
	UUID     string // snapshot uuid (empty for top delta)
	Target   string // mount point (empty if no mount is needed)
	Flags    int    // bit mount flags such as MS_NOATIME
	Data     string // auxiliary mount options
	Readonly bool   // mount read-only
	Fsck     bool   // do fsck before mounting inner FS
	Quota    bool   // enable quota for inner FS
}

// Mount creates a ploop device and (optionally) mounts it
func (d Ploop) Mount(p *MountParam) (string, error) {
	var a C.struct_ploop_mount_param
	var device string

	if p.UUID != "" {
		a.guid = C.CString(p.UUID)
		defer cfree(a.guid)
	}
	if p.Target != "" {
		a.target = C.CString(p.Target)
		defer cfree(a.target)
	}

	// mount_data should not be NULL
	a.mount_data = C.CString(p.Data)
	defer cfree(a.mount_data)

	a.flags = C.int(p.Flags)
	a.ro = bool2cint(p.Readonly)
	a.fsck = bool2cint(p.Fsck)
	a.quota = bool2cint(p.Quota)

	ret := C.ploop_mount_image(d.d, &a)
	if ret == 0 {
		device = C.GoString(&a.device[0])
		// TODO? fsck_code = C.GoString(a.fsck_rc)
	}
	return device, mkerr(ret)
}

// Umount unmounts the ploop filesystem and dismantles the device
func (d Ploop) Umount() error {
	ret := C.ploop_umount_image(d.d)

	return mkerr(ret)
}

// Resize changes the ploop size. Online resize is recommended.
func (d Ploop) Resize(size uint64, offline bool) error {
	var p C.struct_ploop_resize_param

	p.size = convertSize(size)
	p.offline_resize = bool2cint(offline)

	ret := C.ploop_resize_image(d.d, &p)
	return mkerr(ret)
}

// Snapshot creates a ploop snapshot, returning its uuid
func (d Ploop) Snapshot() (string, error) {
	var p C.struct_ploop_snapshot_param
	var uuid, err = UUID()
	if err != nil {
		return "", err
	}
	p.guid = C.CString(uuid)
	defer cfree(p.guid)

	ret := C.ploop_create_snapshot(d.d, &p)
	if ret == 0 {
		uuid = C.GoString(p.guid)
	}

	return uuid, mkerr(ret)
}

// SwitchSnapshot switches to a specified snapshot,
// creates a new empty delta on top of it, and makes it a top one
// (i.e. the one new data will be written to).
// Old top delta (i.e. data modified since the last snapshot) is lost.
func (d Ploop) SwitchSnapshot(uuid string) error {
	var p C.struct_ploop_snapshot_switch_param

	p.guid = C.CString(uuid)
	defer cfree(p.guid)

	ret := C.ploop_switch_snapshot_ex(d.d, &p)

	return mkerr(ret)
}

// Possible values for SwitchSnapshotExtended flags argument
type SwitchFlag uint

const (
	// SkipDestroy, if set, modifies the behavior of
	// SwitchSnapshotExtended to not delete the old top delta, but
	// make it a snapshot and return its uuid. Without this flag,
	// old top delta (i.e. data modified since the last snapshot)
	// is lost.
	SkipDestroy SwitchFlag = C.PLOOP_SNAP_SKIP_TOPDELTA_DESTROY
	// SkipCreate flag, if set, modifies the behavior of
	// SwitchSnapshotExtended to not create a new top delta,
	// but rather transform the specified snapshot itself to be
	// the new top delta), so all new changes will be written
	// right to it. Snapshot UUID is lost in this case.
	SkipCreate SwitchFlag = C.PLOOP_SNAP_SKIP_TOPDELTA_CREATE
)

// SwitchSnapshotExtended is same as SwitchSnapshot but with additional
// flags modifying its behavior. Please see individual flags description.
// Returns uuid of what was the old top delta if SkipDestroy flag is set.
func (d Ploop) SwitchSnapshotExtended(uuid string, flags SwitchFlag) (string, error) {
	var p C.struct_ploop_snapshot_switch_param
	old_uuid := ""

	p.guid = C.CString(uuid)
	defer cfree(p.guid)

	p.flags = C.int(flags)

	if flags&SkipDestroy != 0 {
		old_uuid, err := UUID()
		if err != nil {
			return "", err
		}
		p.guid_old = C.CString(old_uuid)
		defer cfree(p.guid_old)
	}

	ret := C.ploop_switch_snapshot_ex(d.d, &p)

	return old_uuid, mkerr(ret)
}

// DeleteSnapshot deletes a snapshot (merging it down if necessary)
func (d Ploop) DeleteSnapshot(uuid string) error {
	cuuid := C.CString(uuid)
	defer cfree(cuuid)

	ret := C.ploop_delete_snapshot(d.d, cuuid)

	return mkerr(ret)
}

type ReplaceFlag int

// Possible values for ReplaceParam.flags
const (
	// KeepName renames the new file to old file name after replace;
	// note that if this option is used the old file is removed.
	KeepName ReplaceFlag = C.PLOOP_REPLACE_KEEP_NAME
)

// ReplaceParam is a set of parameters to Replace()
type ReplaceParam struct {
	File string // new image file name
	// Image to be replaced is specified by either
	// uuid, current file name, or level,
	// in the above order of preference.
	UUID    string
	CurFile string
	Level   int
	Flags   ReplaceFlag
}

// Replace replaces a ploop image to a different (but identical) one
func (d Ploop) Replace(p *ReplaceParam) error {
	var a C.struct_ploop_replace_param

	a.file = C.CString(p.File)
	defer cfree(a.file)

	if p.UUID != "" {
		a.guid = C.CString(p.UUID)
		defer cfree(a.guid)
	} else if p.CurFile != "" {
		a.cur_file = C.CString(p.CurFile)
		defer cfree(a.cur_file)
	} else {
		a.level = C.int(p.Level)
	}

	a.flags = C.int(p.Flags)

	ret := C.ploop_replace_image(d.d, &a)

	return mkerr(ret)
}

// IsMounted returns true if ploop is mounted
func (d Ploop) IsMounted() (bool, error) {
	ret := C.ploop_is_mounted(d.d)
	if ret == 0 {
		return false, nil
	} else if ret == 1 {
		return true, nil
	} else {
		// error, but no code, make our own
		return false, mkerr(E_SYS)
	}
}

// FSInfoData holds information about ploop inner file system
type FSInfoData struct {
	Blocksize   uint64
	Blocks      uint64
	Blocks_free uint64
	Inodes      uint64
	Inodes_free uint64
}

// FSInfo gets info of ploop's inner file system
func FSInfo(file string) (FSInfoData, error) {
	var cinfo C.struct_ploop_info
	var info FSInfoData
	cfile := C.CString(file)
	defer cfree(cfile)

	once.Do(load_kmod)

	ret := C.ploop_get_info_by_descr(cfile, &cinfo)
	if ret == 0 {
		info.Blocksize = uint64(cinfo.fs_bsize)
		info.Blocks = uint64(cinfo.fs_blocks)
		info.Blocks_free = uint64(cinfo.fs_bfree)
		info.Inodes = uint64(cinfo.fs_inodes)
		info.Inodes_free = uint64(cinfo.fs_ifree)
	}

	return info, mkerr(ret)
}

// ImageInfoData holds information about ploop image
type ImageInfoData struct {
	Blocks    uint64
	Blocksize uint32
	Version   int
}

// ImageInfo gets information about a ploop image
func (d Ploop) ImageInfo() (ImageInfoData, error) {
	var cinfo C.struct_ploop_spec
	var info ImageInfoData

	ret := C.ploop_get_spec(d.d, &cinfo)
	if ret == 0 {
		info.Blocks = uint64(cinfo.size)
		info.Blocksize = uint32(cinfo.blocksize)
		info.Version = int(cinfo.fmt_version)
	}

	return info, mkerr(ret)
}

// TopDeltaFile returns file name of top delta
func (d Ploop) TopDeltaFile() (string, error) {
	const len = 4096 // PATH_MAX
	var out [len]C.char

	ret := C.ploop_get_top_delta_fname(d.d, &out[0], len)
	if ret != 0 {
		// error, but no code, make our own
		return "", mkerr(E_SYS)
	}

	file := C.GoString(&out[0])
	return file, nil
}

// UUID generates a ploop UUID
func UUID() (string, error) {
	var cuuid [39]C.char

	ret := C.ploop_uuid_generate(&cuuid[0], 39)
	if ret != 0 {
		return "", mkerr(ret)
	}

	uuid := C.GoString(&cuuid[0])
	return uuid, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package webhook implements the audit.Backend interface using HTTP webhooks.
package webhook

import (
	"fmt"
	"strings"
	"time"

	"github.com/golang/glog"

	"k8s.io/apimachinery/pkg/apimachinery/announced"
	"k8s.io/apimachinery/pkg/apimachinery/registered"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/conversion"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/runtime"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	"k8s.io/apiserver/pkg/apis/audit/install"
	auditv1alpha1 "k8s.io/apiserver/pkg/apis/audit/v1alpha1"
	"k8s.io/apiserver/pkg/audit"
	"k8s.io/apiserver/pkg/util/webhook"
)

const (
	// ModeBatch indicates that the webhook should buffer audit events
	// internally, sending batch updates either once a certain number of
	// events have been received or a certain amount of time has passed.
	ModeBatch = "batch"
	// ModeBlocking causes the webhook to block on every attempt to process
	// a set of events. This causes requests to the API server to wait for a
	// round trip to the external audit service before sending a response.
	ModeBlocking = "blocking"
)

// AllowedModes is the modes known by this webhook.
var AllowedModes = []string{
	ModeBatch,
	ModeBlocking,
}

const (
	// Default configuration values for ModeBatch.
	//
	// TODO(ericchiang): Make these value configurable. Maybe through a
	// kubeconfig extension?
	defaultBatchBufferSize = 1000        // Buffer up to 1000 events before blocking.
	defaultBatchMaxSize    = 100         // Only send 100 events at a time.
	defaultBatchMaxWait    = time.Minute // Send events at least once a minute.
)

// The plugin name reported in error metrics.
const pluginName = "webhook"

// NewBackend returns an audit backend that sends events over HTTP to an external service.
// The mode indicates the caching behavior of the webhook. Either blocking (ModeBlocking)
// or buffered with batch POSTs (ModeBatch).
func NewBackend(kubeConfigFile string, mode string) (audit.Backend, error) {
	switch mode {
	case ModeBatch:
		return newBatchWebhook(kubeConfigFile)
	case ModeBlocking:
		return newBlockingWebhook(kubeConfigFile)
	default:
		return nil, fmt.Errorf("webhook mode %q is not in list of known modes (%s)",
			mode, strings.Join(AllowedModes, ","))
	}
}

var (
	// NOTE: Copied from other webhook implementations
	//
	// Can we make these passable to NewGenericWebhook?
	groupFactoryRegistry = make(announced.APIGroupFactoryRegistry)
	groupVersions        = []schema.GroupVersion{auditv1alpha1.SchemeGroupVersion}
	registry             = registered.NewOrDie("")
)

func init() {
	registry.RegisterVersions(groupVersions)
	if err := registry.EnableVersions(groupVersions...); err != nil {
		panic(fmt.Sprintf("failed to enable version %v", groupVersions))
	}
	install.Install(groupFactoryRegistry, registry, audit.Scheme)
}

func loadWebhook(configFile string) (*webhook.GenericWebhook, error) {
	return webhook.NewGenericWebhook(registry, audit.Codecs, configFile, groupVersions, 0)
}

func newBlockingWebhook(configFile string) (*blockingBackend, error) {
	w, err := loadWebhook(configFile)
	if err != nil {
		return nil, err
	}
	return &blockingBackend{w}, nil
}

type blockingBackend struct {
	w *webhook.GenericWebhook
}

func (b *blockingBackend) Run(stopCh <-chan struct{}) error {
	return nil
}

func (b *blockingBackend) ProcessEvents(ev ...*auditinternal.Event) {
	if err := b.processEvents(ev...); err != nil {
		audit.HandlePluginError(pluginName, err, ev...)
	}
}

func (b *blockingBackend) processEvents(ev ...*auditinternal.Event) error {
	var list auditinternal.EventList
	for _, e := range ev {
		list.Items = append(list.Items, *e)
	}
	// NOTE: No exponential backoff because this is the blocking webhook
	// mode. Any attempts to retry will block API server requests.
	return b.w.RestClient.Post().Body(&list).Do().Error()
}

func newBatchWebhook(configFile string) (*batchBackend, error) {
	w, err := loadWebhook(configFile)
	if err != nil {
		return nil, err
	}

	c := conversion.NewCloner()
	for _, f := range metav1.GetGeneratedDeepCopyFuncs() {
		if err := c.RegisterGeneratedDeepCopyFunc(f); err != nil {
			return nil, fmt.Errorf("registering meta deep copy method: %v", err)
		}
	}
	for _, f := range auditinternal.GetGeneratedDeepCopyFuncs() {
		if err := c.RegisterGeneratedDeepCopyFunc(f); err != nil {
			return nil, fmt.Errorf("registering audit deep copy method: %v", err)
		}
	}

	return &batchBackend{
		w:            w,
		buffer:       make(chan *auditinternal.Event, defaultBatchBufferSize),
		maxBatchSize: defaultBatchMaxSize,
		maxBatchWait: defaultBatchMaxWait,
		cloner:       c,
	}, nil
}

type batchBackend struct {
	w *webhook.GenericWebhook

	// Cloner is used to deep copy events as they are buffered.
	cloner *conversion.Cloner

	// Channel to buffer events in memory before sending them on the webhook.
	buffer chan *auditinternal.Event
	// Maximum number of events that can be sent at once.
	maxBatchSize int
	// Amount of time to wait after sending events before force sending another set.
	//
	// Receiving maxBatchSize events will always trigger a send, regardless of
	// if this amount of time has been reached.
	maxBatchWait time.Duration
}

func (b *batchBackend) Run(stopCh <-chan struct{}) error {
	f := func() {
		// Recover from any panics caused by this method so a panic in the
		// goroutine can't bring down the main routine.
		defer runtime.HandleCrash()

		t := time.NewTimer(b.maxBatchWait)
		defer t.Stop() // Release ticker resources

		b.sendBatchEvents(stopCh, t.C)
	}

	go func() {
		for {
			f()

			select {
			case <-stopCh:
				return
			default:
			}
		}
	}()
	return nil
}

// sendBatchEvents attempts to batch some number of events to the backend. It POSTs events
// in a goroutine and logging any error encountered during the POST.
//
// The following things can cause sendBatchEvents to exit:
//
//   * Some maximum number of events are received.
//   * Timer has passed, all queued events are sent.
//   * StopCh is closed, all queued events are sent.
//
func (b *batchBackend) sendBatchEvents(stopCh <-chan struct{}, timer <-chan time.Time) {
	var events []auditinternal.Event

L:
	for i := 0; i < b.maxBatchSize; i++ {
		select {
		case ev := <-b.buffer:
			events = append(events, *ev)
		case <-timer:
			// Timer has expired. Send whatever events are in the queue.
			break L
		case <-stopCh:
			// Webhook has shut down. Send the last events.
			break L
		}
	}

	if len(events) == 0 {
		return
	}

	list := auditinternal.EventList{Items: events}
	go func() {
		// Execute the webhook POST in a goroutine to keep it from blocking.
		// This lets the webhook continue to drain the queue immediatly.

		defer runtime.HandleCrash()

		err := webhook.WithExponentialBackoff(0, func() error {
			return b.w.RestClient.Post().Body(&list).Do().Error()
		})
		if err != nil {
			impacted := make([]*auditinternal.Event, len(events))
			for i := range events {
				impacted[i] = &events[i]
			}
			audit.HandlePluginError(pluginName, err, impacted...)
		}
	}()
	return
}

func (b *batchBackend) ProcessEvents(ev ...*auditinternal.Event) {
	for i, e := range ev {
		// Per the audit.Backend interface these events are reused after being
		// sent to the Sink. Deep copy and send the copy to the queue.
		event := new(auditinternal.Event)
		if err := auditinternal.DeepCopy_audit_Event(e, event, b.cloner); err != nil {
			glog.Errorf("failed to clone audit event: %v: %#v", err, e)
			return
		}

		select {
		case b.buffer <- event:
		default:
			audit.HandlePluginError(pluginName, fmt.Errorf("audit webhook queue blocked"), ev[i:]...)
			return
		}
	}
}
<commit_msg>deepcopy: misc fixes for static deepcopy compilation<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package webhook implements the audit.Backend interface using HTTP webhooks.
package webhook

import (
	"fmt"
	"strings"
	"time"

	"k8s.io/apimachinery/pkg/apimachinery/announced"
	"k8s.io/apimachinery/pkg/apimachinery/registered"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/conversion"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/runtime"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	"k8s.io/apiserver/pkg/apis/audit/install"
	auditv1alpha1 "k8s.io/apiserver/pkg/apis/audit/v1alpha1"
	"k8s.io/apiserver/pkg/audit"
	"k8s.io/apiserver/pkg/util/webhook"
)

const (
	// ModeBatch indicates that the webhook should buffer audit events
	// internally, sending batch updates either once a certain number of
	// events have been received or a certain amount of time has passed.
	ModeBatch = "batch"
	// ModeBlocking causes the webhook to block on every attempt to process
	// a set of events. This causes requests to the API server to wait for a
	// round trip to the external audit service before sending a response.
	ModeBlocking = "blocking"
)

// AllowedModes is the modes known by this webhook.
var AllowedModes = []string{
	ModeBatch,
	ModeBlocking,
}

const (
	// Default configuration values for ModeBatch.
	//
	// TODO(ericchiang): Make these value configurable. Maybe through a
	// kubeconfig extension?
	defaultBatchBufferSize = 1000        // Buffer up to 1000 events before blocking.
	defaultBatchMaxSize    = 100         // Only send 100 events at a time.
	defaultBatchMaxWait    = time.Minute // Send events at least once a minute.
)

// The plugin name reported in error metrics.
const pluginName = "webhook"

// NewBackend returns an audit backend that sends events over HTTP to an external service.
// The mode indicates the caching behavior of the webhook. Either blocking (ModeBlocking)
// or buffered with batch POSTs (ModeBatch).
func NewBackend(kubeConfigFile string, mode string) (audit.Backend, error) {
	switch mode {
	case ModeBatch:
		return newBatchWebhook(kubeConfigFile)
	case ModeBlocking:
		return newBlockingWebhook(kubeConfigFile)
	default:
		return nil, fmt.Errorf("webhook mode %q is not in list of known modes (%s)",
			mode, strings.Join(AllowedModes, ","))
	}
}

var (
	// NOTE: Copied from other webhook implementations
	//
	// Can we make these passable to NewGenericWebhook?
	groupFactoryRegistry = make(announced.APIGroupFactoryRegistry)
	groupVersions        = []schema.GroupVersion{auditv1alpha1.SchemeGroupVersion}
	registry             = registered.NewOrDie("")
)

func init() {
	registry.RegisterVersions(groupVersions)
	if err := registry.EnableVersions(groupVersions...); err != nil {
		panic(fmt.Sprintf("failed to enable version %v", groupVersions))
	}
	install.Install(groupFactoryRegistry, registry, audit.Scheme)
}

func loadWebhook(configFile string) (*webhook.GenericWebhook, error) {
	return webhook.NewGenericWebhook(registry, audit.Codecs, configFile, groupVersions, 0)
}

func newBlockingWebhook(configFile string) (*blockingBackend, error) {
	w, err := loadWebhook(configFile)
	if err != nil {
		return nil, err
	}
	return &blockingBackend{w}, nil
}

type blockingBackend struct {
	w *webhook.GenericWebhook
}

func (b *blockingBackend) Run(stopCh <-chan struct{}) error {
	return nil
}

func (b *blockingBackend) ProcessEvents(ev ...*auditinternal.Event) {
	if err := b.processEvents(ev...); err != nil {
		audit.HandlePluginError(pluginName, err, ev...)
	}
}

func (b *blockingBackend) processEvents(ev ...*auditinternal.Event) error {
	var list auditinternal.EventList
	for _, e := range ev {
		list.Items = append(list.Items, *e)
	}
	// NOTE: No exponential backoff because this is the blocking webhook
	// mode. Any attempts to retry will block API server requests.
	return b.w.RestClient.Post().Body(&list).Do().Error()
}

func newBatchWebhook(configFile string) (*batchBackend, error) {
	w, err := loadWebhook(configFile)
	if err != nil {
		return nil, err
	}

	c := conversion.NewCloner()
	for _, f := range metav1.GetGeneratedDeepCopyFuncs() {
		if err := c.RegisterGeneratedDeepCopyFunc(f); err != nil {
			return nil, fmt.Errorf("registering meta deep copy method: %v", err)
		}
	}
	for _, f := range auditinternal.GetGeneratedDeepCopyFuncs() {
		if err := c.RegisterGeneratedDeepCopyFunc(f); err != nil {
			return nil, fmt.Errorf("registering audit deep copy method: %v", err)
		}
	}

	return &batchBackend{
		w:            w,
		buffer:       make(chan *auditinternal.Event, defaultBatchBufferSize),
		maxBatchSize: defaultBatchMaxSize,
		maxBatchWait: defaultBatchMaxWait,
		cloner:       c,
	}, nil
}

type batchBackend struct {
	w *webhook.GenericWebhook

	// Cloner is used to deep copy events as they are buffered.
	cloner *conversion.Cloner

	// Channel to buffer events in memory before sending them on the webhook.
	buffer chan *auditinternal.Event
	// Maximum number of events that can be sent at once.
	maxBatchSize int
	// Amount of time to wait after sending events before force sending another set.
	//
	// Receiving maxBatchSize events will always trigger a send, regardless of
	// if this amount of time has been reached.
	maxBatchWait time.Duration
}

func (b *batchBackend) Run(stopCh <-chan struct{}) error {
	f := func() {
		// Recover from any panics caused by this method so a panic in the
		// goroutine can't bring down the main routine.
		defer runtime.HandleCrash()

		t := time.NewTimer(b.maxBatchWait)
		defer t.Stop() // Release ticker resources

		b.sendBatchEvents(stopCh, t.C)
	}

	go func() {
		for {
			f()

			select {
			case <-stopCh:
				return
			default:
			}
		}
	}()
	return nil
}

// sendBatchEvents attempts to batch some number of events to the backend. It POSTs events
// in a goroutine and logging any error encountered during the POST.
//
// The following things can cause sendBatchEvents to exit:
//
//   * Some maximum number of events are received.
//   * Timer has passed, all queued events are sent.
//   * StopCh is closed, all queued events are sent.
//
func (b *batchBackend) sendBatchEvents(stopCh <-chan struct{}, timer <-chan time.Time) {
	var events []auditinternal.Event

L:
	for i := 0; i < b.maxBatchSize; i++ {
		select {
		case ev := <-b.buffer:
			events = append(events, *ev)
		case <-timer:
			// Timer has expired. Send whatever events are in the queue.
			break L
		case <-stopCh:
			// Webhook has shut down. Send the last events.
			break L
		}
	}

	if len(events) == 0 {
		return
	}

	list := auditinternal.EventList{Items: events}
	go func() {
		// Execute the webhook POST in a goroutine to keep it from blocking.
		// This lets the webhook continue to drain the queue immediatly.

		defer runtime.HandleCrash()

		err := webhook.WithExponentialBackoff(0, func() error {
			return b.w.RestClient.Post().Body(&list).Do().Error()
		})
		if err != nil {
			impacted := make([]*auditinternal.Event, len(events))
			for i := range events {
				impacted[i] = &events[i]
			}
			audit.HandlePluginError(pluginName, err, impacted...)
		}
	}()
	return
}

func (b *batchBackend) ProcessEvents(ev ...*auditinternal.Event) {
	for i, e := range ev {
		// Per the audit.Backend interface these events are reused after being
		// sent to the Sink. Deep copy and send the copy to the queue.
		event := e.DeepCopy()

		select {
		case b.buffer <- event:
		default:
			audit.HandlePluginError(pluginName, fmt.Errorf("audit webhook queue blocked"), ev[i:]...)
			return
		}
	}
}
<|endoftext|>


<commit_before>package recycleme

import (
	"io/ioutil"
	"log"
	"strings"
	"testing"
)

var binsJson = `{
  "Bins": [
    {
      "id": 0,
      "Name": "Green Bin"
    },
    {
      "id": 1,
      "Name": "Yellow Bin"
    },
    {
      "id": 2,
      "Name": "White Bin"
    }
  ]
}`

var materialsJson = `{
  "Materials": [
    {
      "id": 0,
      "Name": "Cardboard box",
      "binIds": [1]
    },
    {
      "id": 1,
      "Name": "Plastic foil",
      "binIds": [0]
    },
    {
      "id": 2,
      "Name": "Plastic bottle",
      "binIds": [1]
    },
    {
      "id": 3,
      "Name": "Glass bottle",
      "binIds": [2]
    },
    {
      "id": 4,
      "Name": "Food",
      "binIds": [0]
    },
    {
      "id": 5,
      "Name": "Plastic bottle cap",
      "binIds": [1]
    },
    {
      "id": 6,
      "Name": "Metal bottle cap",
      "binIds": [1]
    },
    {
      "id": 7,
      "Name": "Kleenex",
      "binIds": [0]
    },
    {
      "id": 8,
      "Name": "Plastic box",
      "binIds": [0]
    }
  ]
}`

var packagesJson = `{
  "Packages": [
    {
      "id": 0,
      "EAN": "7613034383808",
      "materialIds": [
        0,
        1,
        4
      ]
    },
    {
      "id": 1,
      "EAN": "5029053038896",
      "materialIds": [
        0,
        7
      ]
    },
    {
      "id": 2,
      "EAN": "3281780874976",
      "materialIds": [
        1,
        8
      ]
    }
  ]
}`

func init() {
	logger := log.New(ioutil.Discard, "", 0)
	LoadBinsJson(strings.NewReader(binsJson), logger)
	LoadMaterialsJson(strings.NewReader(materialsJson), logger)
	LoadPackagesJson(strings.NewReader(packagesJson), logger)
}

func TestPackage(t *testing.T) {
	r := Package{EAN: "7613034383808", Materials: []Material{
		Material{id: 0, Name: "Cardboard box"},
		Material{id: 1, Name: "Plastic foil"},
		Material{id: 4, Name: "Food"},
	}}

	pkg := Packages["7613034383808"]
	if r.EAN != pkg.EAN || len(r.Materials) != len(pkg.Materials) {
		t.Errorf("Packages for %v differ", r.EAN)
	} else {
		for i, m := range r.Materials {
			pkgMaterial := pkg.Materials[i]
			if m.id != pkgMaterial.id || m.Name != pkgMaterial.Name {
				t.Errorf("Material differ for EAN %v: %v vs %v", r.EAN, m, pkgMaterial)
			}
		}

		binNames := []string{"Yellow Bin", "Green Bin", "Green Bin"}
		for i, m := range r.Materials {
			bins := MaterialsToBins[m]
			if len(bins) != 1 {
				t.Errorf("material %v must belongs to 1 bin, found %v", m.Name, len(bins))
			} else {
				if bins[0].Name != binNames[i] {
					t.Errorf("material %v belong to %v, not %v", m.Name, binNames[i], bins[0].Name)
				}
			}
		}
	}
}

func TestProductPackage(t *testing.T) {
	product, err := Scrap("7613034383808")
	if err != nil {
		t.Error(err)
	} else {
		pp := NewProductPackage(*product)
		materials := []Material{
			Material{id: 0, Name: "Cardboard box"},
			Material{id: 1, Name: "Plastic foil"},
			Material{id: 4, Name: "Food"}}
		if pp.Name != "Four à Pierre Royale" || pp.EAN != "7613034383808" ||
			pp.URL != "http://fr.openfoodfacts.org/api/v0/produit/7613034383808.json" ||
			pp.ImageURL != "http://static.openfoodfacts.org/images/products/761/303/438/3808/front.8.400.jpg" {
			t.Errorf("Some attributes are invalid for: %v", pp)
		}

		if len(pp.materials) != len(materials) {
			t.Errorf("Packages for %v differ", pp.EAN)
		} else {
			for i, m := range materials {
				pkgMaterial := pp.materials[i]
				if m.id != pkgMaterial.id || m.Name != pkgMaterial.Name {
					t.Errorf("Material differ for EAN %v: %v vs %v", pp.EAN, m, pkgMaterial)
				}
			}

			binNames := map[string]string{"Cardboard box": "Yellow Bin", "Plastic foil": "Green Bin", "Food": "Green Bin"}
			i := 0
			for m, bins := range pp.ThrowAway() {
				if len(bins) != 1 {
					t.Errorf("material %v must belongs to 1 bin, found %v", m.Name, len(bins))
				} else {
					if bins[0].Name != binNames[m.Name] {
						t.Errorf("material %v belong to %v, not %v", m.Name, binNames[m.Name], bins[0].Name)
					}
				}
				i++
			}
		}
	}
}

func TestThrowAwayJson(t *testing.T) {
	product, err := Scrap("7613034383808")
	if err != nil {
		t.Error(err)
	} else {
		pkg := NewProductPackage(*product)
		expected := `{"Cardboard box":[{"Name":"Yellow Bin"}],"Food":[{"Name":"Green Bin"}],"Plastic foil":[{"Name":"Green Bin"}]}`
		out, err := pkg.ThrowAwayJson()
		if err != nil {
			t.Error(err)
		} else {

			if string(out) != expected {
				t.Errorf("ThrowAwayJson not as expected: %v %v", string(out), expected)
			}
		}
	}
}
<commit_msg>Smaller tests<commit_after>package recycleme

import (
	"io/ioutil"
	"log"
	"strings"
	"testing"
)

var binsJson = `{
  "Bins": [
    {
      "id": 0,
      "Name": "Green Bin"
    },
    {
      "id": 1,
      "Name": "Yellow Bin"
    },
    {
      "id": 2,
      "Name": "White Bin"
    }
  ]
}`

var materialsJson = `{
  "Materials": [
    {
      "id": 0,
      "Name": "Cardboard box",
      "binIds": [1]
    },
    {
      "id": 1,
      "Name": "Plastic foil",
      "binIds": [0]
    },
    {
      "id": 2,
      "Name": "Plastic bottle",
      "binIds": [1]
    },
    {
      "id": 3,
      "Name": "Glass bottle",
      "binIds": [2]
    },
    {
      "id": 4,
      "Name": "Food",
      "binIds": [0]
    },
    {
      "id": 5,
      "Name": "Plastic bottle cap",
      "binIds": [1]
    },
    {
      "id": 6,
      "Name": "Metal bottle cap",
      "binIds": [1]
    },
    {
      "id": 7,
      "Name": "Kleenex",
      "binIds": [0]
    },
    {
      "id": 8,
      "Name": "Plastic box",
      "binIds": [0]
    }
  ]
}`

var packagesJson = `{
  "Packages": [
    {
      "id": 0,
      "EAN": "7613034383808",
      "materialIds": [
        0,
        1,
        4
      ]
    },
    {
      "id": 1,
      "EAN": "5029053038896",
      "materialIds": [
        0,
        7
      ]
    },
    {
      "id": 2,
      "EAN": "3281780874976",
      "materialIds": [
        1,
        8
      ]
    }
  ]
}`

func init() {
	logger := log.New(ioutil.Discard, "", 0)
	LoadBinsJson(strings.NewReader(binsJson), logger)
	LoadMaterialsJson(strings.NewReader(materialsJson), logger)
	LoadPackagesJson(strings.NewReader(packagesJson), logger)
}

func TestPackage(t *testing.T) {
	r := Package{EAN: "7613034383808", Materials: []Material{
		Material{id: 0, Name: "Cardboard box"},
		Material{id: 1, Name: "Plastic foil"},
		Material{id: 4, Name: "Food"},
	}}

	pkg := Packages["7613034383808"]
	if r.EAN != pkg.EAN || len(r.Materials) != len(pkg.Materials) {
		t.Errorf("Packages for %v differ", r.EAN)
	} else {
		for i, m := range r.Materials {
			pkgMaterial := pkg.Materials[i]
			if m.id != pkgMaterial.id || m.Name != pkgMaterial.Name {
				t.Errorf("Material differ for EAN %v: %v vs %v", r.EAN, m, pkgMaterial)
			}
		}

		binNames := []string{"Yellow Bin", "Green Bin", "Green Bin"}
		for i, m := range r.Materials {
			bins := MaterialsToBins[m]
			if len(bins) != 1 {
				t.Errorf("material %v must belongs to 1 bin, found %v", m.Name, len(bins))
			} else {
				if bins[0].Name != binNames[i] {
					t.Errorf("material %v belong to %v, not %v", m.Name, binNames[i], bins[0].Name)
				}
			}
		}
	}
}

func TestProductPackage(t *testing.T) {
	product := Product{EAN: "7613034383808", Name: "Four à Pierre Royale", URL: "http://fr.openfoodfacts.org/api/v0/produit/7613034383808.json", ImageURL: "http://static.openfoodfacts.org/images/products/761/303/438/3808/front.8.400.jpg"}
	pp := NewProductPackage(product)
	materials := []Material{
		Material{id: 0, Name: "Cardboard box"},
		Material{id: 1, Name: "Plastic foil"},
		Material{id: 4, Name: "Food"}}
	if pp.Product != product {
		t.Errorf("Some attributes are invalid for: %v; expected %v", pp, product)
	}

	if len(pp.materials) != len(materials) {
		t.Errorf("Packages for %v differ", pp.EAN)
	} else {
		for i, m := range materials {
			pkgMaterial := pp.materials[i]
			if m.id != pkgMaterial.id || m.Name != pkgMaterial.Name {
				t.Errorf("Material differ for EAN %v: %v vs %v", pp.EAN, m, pkgMaterial)
			}
		}

		binNames := map[string]string{"Cardboard box": "Yellow Bin", "Plastic foil": "Green Bin", "Food": "Green Bin"}
		i := 0
		for m, bins := range pp.ThrowAway() {
			if len(bins) != 1 {
				t.Errorf("material %v must belongs to 1 bin, found %v", m.Name, len(bins))
			} else {
				if bins[0].Name != binNames[m.Name] {
					t.Errorf("material %v belong to %v, not %v", m.Name, binNames[m.Name], bins[0].Name)
				}
			}
			i++
		}
	}
}

func TestThrowAwayJson(t *testing.T) {
	product := Product{EAN: "7613034383808"}
	pkg := NewProductPackage(product)
	expected := `{"Cardboard box":[{"Name":"Yellow Bin"}],"Food":[{"Name":"Green Bin"}],"Plastic foil":[{"Name":"Green Bin"}]}`
	out, err := pkg.ThrowAwayJson()
	if err != nil {
		t.Error(err)
	} else {
		if string(out) != expected {
			t.Errorf("ThrowAwayJson not as expected: %v %v", string(out), expected)
		}
	}
}
<|endoftext|>


<commit_before>package httpjson

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/inputs"
	"github.com/influxdata/telegraf/plugins/parsers"
)

// HttpJson struct
type HttpJson struct {
	Name            string
	Namespace       string
	Servers         []string
	Method          string
	TagKeys         []string
	ResponseTimeout internal.Duration
	Parameters      map[string]string
	Headers         map[string]string

	// Path to CA file
	SSLCA string `toml:"ssl_ca"`
	// Path to host cert file
	SSLCert string `toml:"ssl_cert"`
	// Path to cert key file
	SSLKey string `toml:"ssl_key"`
	// Use SSL but skip chain & host verification
	InsecureSkipVerify bool

	client HTTPClient
}

type HTTPClient interface {
	// Returns the result of an http request
	//
	// Parameters:
	// req: HTTP request object
	//
	// Returns:
	// http.Response:  HTTP respons object
	// error        :  Any error that may have occurred
	MakeRequest(req *http.Request) (*http.Response, error)

	SetHTTPClient(client *http.Client)
	HTTPClient() *http.Client
}

type RealHTTPClient struct {
	client *http.Client
}

func (c *RealHTTPClient) MakeRequest(req *http.Request) (*http.Response, error) {
	return c.client.Do(req)
}

func (c *RealHTTPClient) SetHTTPClient(client *http.Client) {
	c.client = client
}

func (c *RealHTTPClient) HTTPClient() *http.Client {
	return c.client
}

var sampleConfig = `
  ## NOTE This plugin only reads numerical measurements, strings and booleans
  ## will be ignored.

  ## a name for the service being polled
  name = "webserver_stats"
  ## a namespace used as as extra tag in measurement
  namespace = "bbox"

  ## URL of each server in the service's cluster
  servers = [
    "http://localhost:9999/stats/",
    "http://localhost:9998/stats/",
  ]
  ## Set response_timeout (default 5 seconds)
  response_timeout = "5s"

  ## HTTP method to use: GET or POST (case-sensitive)
  method = "GET"

  ## List of tag names to extract from top-level of JSON server response
  # tag_keys = [
  #   "my_tag_1",
  #   "my_tag_2"
  # ]

  ## HTTP parameters (all values must be strings)
  [inputs.httpjson.parameters]
    event_type = "cpu_spike"
    threshold = "0.75"

  ## HTTP Header parameters (all values must be strings)
  # [inputs.httpjson.headers]
  #   X-Auth-Token = "my-xauth-token"
  #   apiVersion = "v1"

  ## Optional SSL Config
  # ssl_ca = "/etc/telegraf/ca.pem"
  # ssl_cert = "/etc/telegraf/cert.pem"
  # ssl_key = "/etc/telegraf/key.pem"
  ## Use SSL but skip chain & host verification
  # insecure_skip_verify = false
`

func (h *HttpJson) SampleConfig() string {
	return sampleConfig
}

func (h *HttpJson) Description() string {
	return "Read flattened metrics from one or more JSON HTTP endpoints"
}

// Gathers data for all servers.
func (h *HttpJson) Gather(acc telegraf.Accumulator) error {
	var wg sync.WaitGroup

	if h.client.HTTPClient() == nil {
		tlsCfg, err := internal.GetTLSConfig(
			h.SSLCert, h.SSLKey, h.SSLCA, h.InsecureSkipVerify)
		if err != nil {
			return err
		}
		tr := &http.Transport{
			ResponseHeaderTimeout: h.ResponseTimeout.Duration,
			TLSClientConfig:       tlsCfg,
		}
		client := &http.Client{
			Transport: tr,
			Timeout:   h.ResponseTimeout.Duration,
		}
		h.client.SetHTTPClient(client)
	}

	errorChannel := make(chan error, len(h.Servers))

	for _, server := range h.Servers {
		wg.Add(1)
		go func(server string) {
			defer wg.Done()
			if err := h.gatherServer(acc, server); err != nil {
				errorChannel <- err
			}
		}(server)
	}

	wg.Wait()
	close(errorChannel)

	// Get all errors and return them as one giant error
	errorStrings := []string{}
	for err := range errorChannel {
		errorStrings = append(errorStrings, err.Error())
	}

	if len(errorStrings) == 0 {
		return nil
	}
	return errors.New(strings.Join(errorStrings, "\n"))
}

// Gathers data from a particular server
// Parameters:
//     acc      : The telegraf Accumulator to use
//     serverURL: endpoint to send request to
//     service  : the service being queried
//
// Returns:
//     error: Any error that may have occurred
func (h *HttpJson) gatherServer(
	acc telegraf.Accumulator,
	serverURL string,
) error {
	resp, _, err := h.sendRequest(serverURL)

	if err != nil {
		return err
	}

	var msrmnt_name string
	if h.Name == "" {
		msrmnt_name = "httpjson"
	} else {
		msrmnt_name = "httpjson_" + h.Name
	}
	tags := map[string]string{
		"server":    serverURL,
		"namespace": h.Namespace,
	}

	parser, err := parsers.NewJSONParser(msrmnt_name, h.TagKeys, tags)
	if err != nil {
		return err
	}

	metrics, err := parser.Parse([]byte(resp))
	if err != nil {
		return err
	}

	for _, metric := range metrics {
		metricGroups := make(map[string]map[string]interface{})

		for fieldName, fieldValue := range metric.Fields() {
			lastDot := strings.LastIndex(fieldName, ".")
			newMetricGroupName := fieldName
			newFieldName := "value"

			if lastDot > 0 {
				newMetricGroupName = metric.Name() + "." + fieldName[:lastDot]
				newFieldName = fieldName[lastDot+1 : len(fieldName)]
			}

			if newFieldName == "time" {
				newFieldName = "timeValue"
			}

			add(metricGroups, newMetricGroupName, newFieldName, fieldValue)
		}

		for metricGroupName, fields := range metricGroups {
			acc.AddFields(metricGroupName, fields, metric.Tags())
		}

	}
	return nil
}

func add(m map[string]map[string]interface{}, metricGroupName, fieldName string, fieldValue interface{}) {
	mm, ok := m[metricGroupName]
	if !ok {
		mm = make(map[string]interface{})
		m[metricGroupName] = mm
	}
	mm[fieldName] = fieldValue
}

// Sends an HTTP request to the server using the HttpJson object's HTTPClient.
// This request can be either a GET or a POST.
// Parameters:
//     serverURL: endpoint to send request to
//
// Returns:
//     string: body of the response
//     error : Any error that may have occurred
func (h *HttpJson) sendRequest(serverURL string) (string, float64, error) {
	// Prepare URL
	requestURL, err := url.Parse(serverURL)
	if err != nil {
		return "", -1, fmt.Errorf("Invalid server URL \"%s\"", serverURL)
	}

	data := url.Values{}
	switch {
	case h.Method == "GET":
		params := requestURL.Query()
		for k, v := range h.Parameters {
			params.Add(k, v)
		}
		requestURL.RawQuery = params.Encode()

	case h.Method == "POST":
		requestURL.RawQuery = ""
		for k, v := range h.Parameters {
			data.Add(k, v)
		}
	}

	// Create + send request
	req, err := http.NewRequest(h.Method, requestURL.String(),
		strings.NewReader(data.Encode()))
	if err != nil {
		return "", -1, err
	}

	// Add header parameters
	for k, v := range h.Headers {
		if strings.ToLower(k) == "host" {
			req.Host = v
		} else {
			req.Header.Add(k, v)
		}
	}

	start := time.Now()
	resp, err := h.client.MakeRequest(req)
	if err != nil {
		return "", -1, err
	}

	defer resp.Body.Close()
	responseTime := time.Since(start).Seconds()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return string(body), responseTime, err
	}

	// Process response
	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("Response from url \"%s\" has status code %d (%s), expected %d (%s)",
			requestURL.String(),
			resp.StatusCode,
			http.StatusText(resp.StatusCode),
			http.StatusOK,
			http.StatusText(http.StatusOK))
		return string(body), responseTime, err
	}

	return string(body), responseTime, err
}

func init() {
	inputs.Add("httpjson", func() telegraf.Input {
		return &HttpJson{
			client: &RealHTTPClient{},
			ResponseTimeout: internal.Duration{
				Duration: 5 * time.Second,
			},
		}
	})
}
<commit_msg>single metrics without a dot<commit_after>package httpjson

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/inputs"
	"github.com/influxdata/telegraf/plugins/parsers"
)

// HttpJson struct
type HttpJson struct {
	Name            string
	Namespace       string
	Servers         []string
	Method          string
	TagKeys         []string
	ResponseTimeout internal.Duration
	Parameters      map[string]string
	Headers         map[string]string

	// Path to CA file
	SSLCA string `toml:"ssl_ca"`
	// Path to host cert file
	SSLCert string `toml:"ssl_cert"`
	// Path to cert key file
	SSLKey string `toml:"ssl_key"`
	// Use SSL but skip chain & host verification
	InsecureSkipVerify bool

	client HTTPClient
}

type HTTPClient interface {
	// Returns the result of an http request
	//
	// Parameters:
	// req: HTTP request object
	//
	// Returns:
	// http.Response:  HTTP respons object
	// error        :  Any error that may have occurred
	MakeRequest(req *http.Request) (*http.Response, error)

	SetHTTPClient(client *http.Client)
	HTTPClient() *http.Client
}

type RealHTTPClient struct {
	client *http.Client
}

func (c *RealHTTPClient) MakeRequest(req *http.Request) (*http.Response, error) {
	return c.client.Do(req)
}

func (c *RealHTTPClient) SetHTTPClient(client *http.Client) {
	c.client = client
}

func (c *RealHTTPClient) HTTPClient() *http.Client {
	return c.client
}

var sampleConfig = `
  ## NOTE This plugin only reads numerical measurements, strings and booleans
  ## will be ignored.

  ## a name for the service being polled
  name = "webserver_stats"
  ## a namespace used as as extra tag in measurement
  namespace = "bbox"

  ## URL of each server in the service's cluster
  servers = [
    "http://localhost:9999/stats/",
    "http://localhost:9998/stats/",
  ]
  ## Set response_timeout (default 5 seconds)
  response_timeout = "5s"

  ## HTTP method to use: GET or POST (case-sensitive)
  method = "GET"

  ## List of tag names to extract from top-level of JSON server response
  # tag_keys = [
  #   "my_tag_1",
  #   "my_tag_2"
  # ]

  ## HTTP parameters (all values must be strings)
  [inputs.httpjson.parameters]
    event_type = "cpu_spike"
    threshold = "0.75"

  ## HTTP Header parameters (all values must be strings)
  # [inputs.httpjson.headers]
  #   X-Auth-Token = "my-xauth-token"
  #   apiVersion = "v1"

  ## Optional SSL Config
  # ssl_ca = "/etc/telegraf/ca.pem"
  # ssl_cert = "/etc/telegraf/cert.pem"
  # ssl_key = "/etc/telegraf/key.pem"
  ## Use SSL but skip chain & host verification
  # insecure_skip_verify = false
`

func (h *HttpJson) SampleConfig() string {
	return sampleConfig
}

func (h *HttpJson) Description() string {
	return "Read flattened metrics from one or more JSON HTTP endpoints"
}

// Gathers data for all servers.
func (h *HttpJson) Gather(acc telegraf.Accumulator) error {
	var wg sync.WaitGroup

	if h.client.HTTPClient() == nil {
		tlsCfg, err := internal.GetTLSConfig(
			h.SSLCert, h.SSLKey, h.SSLCA, h.InsecureSkipVerify)
		if err != nil {
			return err
		}
		tr := &http.Transport{
			ResponseHeaderTimeout: h.ResponseTimeout.Duration,
			TLSClientConfig:       tlsCfg,
		}
		client := &http.Client{
			Transport: tr,
			Timeout:   h.ResponseTimeout.Duration,
		}
		h.client.SetHTTPClient(client)
	}

	errorChannel := make(chan error, len(h.Servers))

	for _, server := range h.Servers {
		wg.Add(1)
		go func(server string) {
			defer wg.Done()
			if err := h.gatherServer(acc, server); err != nil {
				errorChannel <- err
			}
		}(server)
	}

	wg.Wait()
	close(errorChannel)

	// Get all errors and return them as one giant error
	errorStrings := []string{}
	for err := range errorChannel {
		errorStrings = append(errorStrings, err.Error())
	}

	if len(errorStrings) == 0 {
		return nil
	}
	return errors.New(strings.Join(errorStrings, "\n"))
}

// Gathers data from a particular server
// Parameters:
//     acc      : The telegraf Accumulator to use
//     serverURL: endpoint to send request to
//     service  : the service being queried
//
// Returns:
//     error: Any error that may have occurred
func (h *HttpJson) gatherServer(
	acc telegraf.Accumulator,
	serverURL string,
) error {
	resp, _, err := h.sendRequest(serverURL)

	if err != nil {
		return err
	}

	var msrmnt_name string
	if h.Name == "" {
		msrmnt_name = "httpjson"
	} else {
		msrmnt_name = "httpjson_" + h.Name
	}
	tags := map[string]string{
		"server":    serverURL,
		"namespace": h.Namespace,
	}

	parser, err := parsers.NewJSONParser(msrmnt_name, h.TagKeys, tags)
	if err != nil {
		return err
	}

	metrics, err := parser.Parse([]byte(resp))
	if err != nil {
		return err
	}

	for _, metric := range metrics {
		metricGroups := make(map[string]map[string]interface{})

		for fieldName, fieldValue := range metric.Fields() {
			lastDot := strings.LastIndex(fieldName, ".")
			newMetricGroupName := fieldName
			newFieldName := "value"

			if lastDot > 0 {
				newMetricGroupName = metric.Name() + "." + fieldName[:lastDot]
				newFieldName = fieldName[lastDot+1 : len(fieldName)]
			} else {
				newMetricGroupName = metric.Name() + "." + newFieldName
			}

			if newFieldName == "time" {
				newFieldName = "timeValue"
			}

			add(metricGroups, newMetricGroupName, newFieldName, fieldValue)
		}

		for metricGroupName, fields := range metricGroups {
			acc.AddFields(metricGroupName, fields, metric.Tags())
		}

	}
	return nil
}

func add(m map[string]map[string]interface{}, metricGroupName, fieldName string, fieldValue interface{}) {
	mm, ok := m[metricGroupName]
	if !ok {
		mm = make(map[string]interface{})
		m[metricGroupName] = mm
	}
	mm[fieldName] = fieldValue
}

// Sends an HTTP request to the server using the HttpJson object's HTTPClient.
// This request can be either a GET or a POST.
// Parameters:
//     serverURL: endpoint to send request to
//
// Returns:
//     string: body of the response
//     error : Any error that may have occurred
func (h *HttpJson) sendRequest(serverURL string) (string, float64, error) {
	// Prepare URL
	requestURL, err := url.Parse(serverURL)
	if err != nil {
		return "", -1, fmt.Errorf("Invalid server URL \"%s\"", serverURL)
	}

	data := url.Values{}
	switch {
	case h.Method == "GET":
		params := requestURL.Query()
		for k, v := range h.Parameters {
			params.Add(k, v)
		}
		requestURL.RawQuery = params.Encode()

	case h.Method == "POST":
		requestURL.RawQuery = ""
		for k, v := range h.Parameters {
			data.Add(k, v)
		}
	}

	// Create + send request
	req, err := http.NewRequest(h.Method, requestURL.String(),
		strings.NewReader(data.Encode()))
	if err != nil {
		return "", -1, err
	}

	// Add header parameters
	for k, v := range h.Headers {
		if strings.ToLower(k) == "host" {
			req.Host = v
		} else {
			req.Header.Add(k, v)
		}
	}

	start := time.Now()
	resp, err := h.client.MakeRequest(req)
	if err != nil {
		return "", -1, err
	}

	defer resp.Body.Close()
	responseTime := time.Since(start).Seconds()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return string(body), responseTime, err
	}

	// Process response
	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("Response from url \"%s\" has status code %d (%s), expected %d (%s)",
			requestURL.String(),
			resp.StatusCode,
			http.StatusText(resp.StatusCode),
			http.StatusOK,
			http.StatusText(http.StatusOK))
		return string(body), responseTime, err
	}

	return string(body), responseTime, err
}

func init() {
	inputs.Add("httpjson", func() telegraf.Input {
		return &HttpJson{
			client: &RealHTTPClient{},
			ResponseTimeout: internal.Duration{
				Duration: 5 * time.Second,
			},
		}
	})
}
<|endoftext|>


<commit_before>package param

import (
	"html/template"
	"sync"
	"time"
)

func NewMap() *SafeMap {
	return &SafeMap{}
}

type SafeMap struct {
	sync.Map
}

func (s *SafeMap) Set(key, value interface{}) {
	s.Store(key, value)
}

func (s *SafeMap) Get(key interface{}, defaults ...interface{}) interface{} {
	value, ok := s.Load(key)
	if (!ok || value == nil) && len(defaults) > 0 {
		if fallback, ok := defaults[0].(func() interface{}); ok {
			return fallback()
		}
		return defaults[0]
	}
	return value
}

func (s *SafeMap) GetOk(key interface{}) (interface{}, bool) {
	return s.Load(key)
}

func (s *SafeMap) Has(key interface{}) bool {
	_, ok := s.Load(key)
	return ok
}

func (s *SafeMap) GetOrSet(key, value interface{}) (actual interface{}, loaded bool) {
	return s.LoadOrStore(key, value)
}

func (s *SafeMap) String(key interface{}, defaults ...interface{}) string {
	return AsString(s.Get(key, defaults...))
}

func (s *SafeMap) Split(key interface{}, sep string, limit ...int) StringSlice {
	return Split(s.Get(key), sep, limit...)
}

func (s *SafeMap) Trim(key interface{}, defaults ...interface{}) String {
	return Trim(s.Get(key, defaults...))
}

func (s *SafeMap) HTML(key interface{}, defaults ...interface{}) template.HTML {
	return AsHTML(s.Get(key, defaults...))
}

func (s *SafeMap) HTMLAttr(key interface{}, defaults ...interface{}) template.HTMLAttr {
	return AsHTMLAttr(s.Get(key, defaults...))
}

func (s *SafeMap) JS(key interface{}, defaults ...interface{}) template.JS {
	return AsJS(s.Get(key, defaults...))
}

func (s *SafeMap) CSS(key interface{}, defaults ...interface{}) template.CSS {
	return AsCSS(s.Get(key, defaults...))
}

func (s *SafeMap) Bool(key interface{}, defaults ...interface{}) bool {
	return AsBool(s.Get(key, defaults...))
}

func (s *SafeMap) Float64(key interface{}, defaults ...interface{}) float64 {
	return AsFloat64(s.Get(key, defaults...))
}

func (s *SafeMap) Float32(key interface{}, defaults ...interface{}) float32 {
	return AsFloat32(s.Get(key, defaults...))
}

func (s *SafeMap) Int8(key interface{}, defaults ...interface{}) int8 {
	return AsInt8(s.Get(key, defaults...))
}

func (s *SafeMap) Int16(key interface{}, defaults ...interface{}) int16 {
	return AsInt16(s.Get(key, defaults...))
}

func (s *SafeMap) Int(key interface{}, defaults ...interface{}) int {
	return AsInt(s.Get(key, defaults...))
}

func (s *SafeMap) Int32(key interface{}, defaults ...interface{}) int32 {
	return AsInt32(s.Get(key, defaults...))
}

func (s *SafeMap) Int64(key interface{}, defaults ...interface{}) int64 {
	return AsInt64(s.Get(key, defaults...))
}

func (s *SafeMap) Decr(key interface{}, n int64, defaults ...interface{}) int64 {
	v := Decr(s.Get(key, defaults...), n)
	s.Set(key, v)
	return v
}

func (s *SafeMap) Incr(key interface{}, n int64, defaults ...interface{}) int64 {
	v := Incr(s.Get(key, defaults...), n)
	s.Set(key, v)
	return v
}

func (s *SafeMap) Uint8(key interface{}, defaults ...interface{}) uint8 {
	return AsUint8(s.Get(key, defaults...))
}

func (s *SafeMap) Uint16(key interface{}, defaults ...interface{}) uint16 {
	return AsUint16(s.Get(key, defaults...))
}

func (s *SafeMap) Uint(key interface{}, defaults ...interface{}) uint {
	return AsUint(s.Get(key, defaults...))
}

func (s *SafeMap) Uint32(key interface{}, defaults ...interface{}) uint32 {
	return AsUint32(s.Get(key, defaults...))
}

func (s *SafeMap) Uint64(key interface{}, defaults ...interface{}) uint64 {
	return AsUint64(s.Get(key, defaults...))
}

func (s *SafeMap) Timestamp(key interface{}, defaults ...interface{}) time.Time {
	return AsTimestamp(s.Get(key, defaults...))
}

func (s *SafeMap) DateTime(key interface{}, layouts ...string) time.Time {
	return AsDateTime(s.Get(key), layouts...)
}
<commit_msg>update<commit_after>package param

import (
	"html/template"
	"sync"
	"time"
)

func NewMap() *SafeMap {
	return &SafeMap{}
}

type SafeMap struct {
	sync.Map
}

func (s *SafeMap) Set(key, value interface{}) {
	s.Store(key, value)
}

func (s *SafeMap) Get(key interface{}, defaults ...interface{}) interface{} {
	value, ok := s.Load(key)
	if (!ok || value == nil) && len(defaults) > 0 {
		if fallback, ok := defaults[0].(func() interface{}); ok {
			return fallback()
		}
		return defaults[0]
	}
	return value
}

func (s *SafeMap) GetOk(key interface{}) (interface{}, bool) {
	return s.Load(key)
}

func (s *SafeMap) Has(key interface{}) bool {
	_, ok := s.Load(key)
	return ok
}

func (s *SafeMap) GetOrSet(key, value interface{}) (actual interface{}, loaded bool) {
	return s.LoadOrStore(key, value)
}

func (s *SafeMap) String(key interface{}, defaults ...interface{}) string {
	return AsString(s.Get(key, defaults...))
}

func (s *SafeMap) Split(key interface{}, sep string, limit ...int) StringSlice {
	return Split(s.Get(key), sep, limit...)
}

func (s *SafeMap) Trim(key interface{}, defaults ...interface{}) String {
	return Trim(s.Get(key, defaults...))
}

func (s *SafeMap) HTML(key interface{}, defaults ...interface{}) template.HTML {
	return AsHTML(s.Get(key, defaults...))
}

func (s *SafeMap) HTMLAttr(key interface{}, defaults ...interface{}) template.HTMLAttr {
	return AsHTMLAttr(s.Get(key, defaults...))
}

func (s *SafeMap) JS(key interface{}, defaults ...interface{}) template.JS {
	return AsJS(s.Get(key, defaults...))
}

func (s *SafeMap) CSS(key interface{}, defaults ...interface{}) template.CSS {
	return AsCSS(s.Get(key, defaults...))
}

func (s *SafeMap) Bool(key interface{}, defaults ...interface{}) bool {
	return AsBool(s.Get(key, defaults...))
}

func (s *SafeMap) Float64(key interface{}, defaults ...interface{}) float64 {
	return AsFloat64(s.Get(key, defaults...))
}

func (s *SafeMap) Float32(key interface{}, defaults ...interface{}) float32 {
	return AsFloat32(s.Get(key, defaults...))
}

func (s *SafeMap) Int8(key interface{}, defaults ...interface{}) int8 {
	return AsInt8(s.Get(key, defaults...))
}

func (s *SafeMap) Int16(key interface{}, defaults ...interface{}) int16 {
	return AsInt16(s.Get(key, defaults...))
}

func (s *SafeMap) Int(key interface{}, defaults ...interface{}) int {
	return AsInt(s.Get(key, defaults...))
}

func (s *SafeMap) Int32(key interface{}, defaults ...interface{}) int32 {
	return AsInt32(s.Get(key, defaults...))
}

func (s *SafeMap) Int64(key interface{}, defaults ...interface{}) int64 {
	return AsInt64(s.Get(key, defaults...))
}

func (s *SafeMap) Decr(key interface{}, n int64, defaults ...interface{}) int64 {
	v := Decr(s.Get(key, defaults...), n)
	s.Set(key, v)
	return v
}

func (s *SafeMap) Incr(key interface{}, n int64, defaults ...interface{}) int64 {
	v := Incr(s.Get(key, defaults...), n)
	s.Set(key, v)
	return v
}

func (s *SafeMap) Uint8(key interface{}, defaults ...interface{}) uint8 {
	return AsUint8(s.Get(key, defaults...))
}

func (s *SafeMap) Uint16(key interface{}, defaults ...interface{}) uint16 {
	return AsUint16(s.Get(key, defaults...))
}

func (s *SafeMap) Uint(key interface{}, defaults ...interface{}) uint {
	return AsUint(s.Get(key, defaults...))
}

func (s *SafeMap) Uint32(key interface{}, defaults ...interface{}) uint32 {
	return AsUint32(s.Get(key, defaults...))
}

func (s *SafeMap) Uint64(key interface{}, defaults ...interface{}) uint64 {
	return AsUint64(s.Get(key, defaults...))
}

func (s *SafeMap) GetStore(key interface{}, defaults ...interface{}) Store {
	return AsStore(s.Get(key, defaults...))
}

func (s *SafeMap) Timestamp(key interface{}, defaults ...interface{}) time.Time {
	return AsTimestamp(s.Get(key, defaults...))
}

func (s *SafeMap) DateTime(key interface{}, layouts ...string) time.Time {
	return AsDateTime(s.Get(key), layouts...)
}
<|endoftext|>


<commit_before>package csimanager

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"time"

	grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/helper/mount"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/plugins/csi"
)

var _ VolumeMounter = &volumeManager{}

const (
	DefaultMountActionTimeout = 2 * time.Minute
	StagingDirName            = "staging"
	AllocSpecificDirName      = "per-alloc"
)

// volumeManager handles the state of attached volumes for a given CSI Plugin.
//
// volumeManagers outlive the lifetime of a given allocation as volumes may be
// shared by multiple allocations on the same node.
//
// volumes are stored by an enriched volume usage struct as the CSI Spec requires
// slightly different usage based on the given usage model.
type volumeManager struct {
	logger hclog.Logger
	plugin csi.CSIPlugin

	usageTracker *volumeUsageTracker

	// mountRoot is the root of where plugin directories and mounts may be created
	// e.g /opt/nomad.d/statedir/csi/my-csi-plugin/
	mountRoot string

	// containerMountPoint is the location _inside_ the plugin container that the
	// `mountRoot` is bound in to.
	containerMountPoint string

	// requiresStaging shows whether the plugin requires that the volume manager
	// calls NodeStageVolume and NodeUnstageVolume RPCs during setup and teardown
	requiresStaging bool
}

func newVolumeManager(logger hclog.Logger, plugin csi.CSIPlugin, rootDir, containerRootDir string, requiresStaging bool) *volumeManager {
	return &volumeManager{
		logger:              logger.Named("volume_manager"),
		plugin:              plugin,
		mountRoot:           rootDir,
		containerMountPoint: containerRootDir,
		requiresStaging:     requiresStaging,
		usageTracker:        newVolumeUsageTracker(),
	}
}

func (v *volumeManager) stagingDirForVolume(root string, vol *structs.CSIVolume, usage *UsageOptions) string {
	return filepath.Join(root, StagingDirName, vol.ID, usage.ToFS())
}

func (v *volumeManager) allocDirForVolume(root string, vol *structs.CSIVolume, alloc *structs.Allocation, usage *UsageOptions) string {
	return filepath.Join(root, AllocSpecificDirName, alloc.ID, vol.ID, usage.ToFS())
}

// ensureStagingDir attempts to create a directory for use when staging a volume
// and then validates that the path is not already a mount point for e.g an
// existing volume stage.
//
// Returns whether the directory is a pre-existing mountpoint, the staging path,
// and any errors that occurred.
func (v *volumeManager) ensureStagingDir(vol *structs.CSIVolume, usage *UsageOptions) (string, bool, error) {
	stagingPath := v.stagingDirForVolume(v.mountRoot, vol, usage)

	// Make the staging path, owned by the Nomad User
	if err := os.MkdirAll(stagingPath, 0700); err != nil && !os.IsExist(err) {
		return "", false, fmt.Errorf("failed to create staging directory for volume (%s): %v", vol.ID, err)

	}

	// Validate that it is not already a mount point
	m := mount.New()
	isNotMount, err := m.IsNotAMountPoint(stagingPath)
	if err != nil {
		return "", false, fmt.Errorf("mount point detection failed for volume (%s): %v", vol.ID, err)
	}

	return stagingPath, !isNotMount, nil
}

// ensureAllocDir attempts to create a directory for use when publishing a volume
// and then validates that the path is not already a mount point (e.g when reattaching
// to existing allocs).
//
// Returns whether the directory is a pre-existing mountpoint, the publish path,
// and any errors that occurred.
func (v *volumeManager) ensureAllocDir(vol *structs.CSIVolume, alloc *structs.Allocation, usage *UsageOptions) (string, bool, error) {
	allocPath := v.allocDirForVolume(v.mountRoot, vol, alloc, usage)

	// Make the alloc path, owned by the Nomad User
	if err := os.MkdirAll(allocPath, 0700); err != nil && !os.IsExist(err) {
		return "", false, fmt.Errorf("failed to create allocation directory for volume (%s): %v", vol.ID, err)
	}

	// Validate that it is not already a mount point
	m := mount.New()
	isNotMount, err := m.IsNotAMountPoint(allocPath)
	if err != nil {
		return "", false, fmt.Errorf("mount point detection failed for volume (%s): %v", vol.ID, err)
	}

	return allocPath, !isNotMount, nil
}

// stageVolume prepares a volume for use by allocations. When a plugin exposes
// the STAGE_UNSTAGE_VOLUME capability it MUST be called once-per-volume for a
// given usage mode before the volume can be NodePublish-ed.
func (v *volumeManager) stageVolume(ctx context.Context, vol *structs.CSIVolume, usage *UsageOptions, publishContext map[string]string) error {
	logger := hclog.FromContext(ctx)
	logger.Trace("Preparing volume staging environment")
	hostStagingPath, isMount, err := v.ensureStagingDir(vol, usage)
	if err != nil {
		return err
	}
	pluginStagingPath := v.stagingDirForVolume(v.containerMountPoint, vol, usage)

	logger.Trace("Volume staging environment", "pre-existing_mount", isMount, "host_staging_path", hostStagingPath, "plugin_staging_path", pluginStagingPath)

	if isMount {
		logger.Debug("re-using existing staging mount for volume", "staging_path", hostStagingPath)
		return nil
	}

	capability, err := csi.VolumeCapabilityFromStructs(vol.AttachmentMode, vol.AccessMode)
	if err != nil {
		return err
	}

	// We currently treat all explicit CSI NodeStageVolume errors (aside from timeouts, codes.ResourceExhausted, and codes.Unavailable)
	// as fatal.
	// In the future, we can provide more useful error messages based on
	// different types of error. For error documentation see:
	// https://github.com/container-storage-interface/spec/blob/4731db0e0bc53238b93850f43ab05d9355df0fd9/spec.md#nodestagevolume-errors
	return v.plugin.NodeStageVolume(ctx,
		vol.ID,
		publishContext,
		pluginStagingPath,
		capability,
		grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
		grpc_retry.WithMax(3),
		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)),
	)
}

func (v *volumeManager) publishVolume(ctx context.Context, vol *structs.CSIVolume, alloc *structs.Allocation, usage *UsageOptions, publishContext map[string]string) (*MountInfo, error) {
	logger := hclog.FromContext(ctx)
	var pluginStagingPath string
	if v.requiresStaging {
		pluginStagingPath = v.stagingDirForVolume(v.containerMountPoint, vol, usage)
	}

	hostTargetPath, isMount, err := v.ensureAllocDir(vol, alloc, usage)
	if err != nil {
		return nil, err
	}
	pluginTargetPath := v.allocDirForVolume(v.containerMountPoint, vol, alloc, usage)

	if isMount {
		logger.Debug("Re-using existing published volume for allocation")
		return &MountInfo{Source: hostTargetPath}, nil
	}

	capabilities, err := csi.VolumeCapabilityFromStructs(vol.AttachmentMode, vol.AccessMode)
	if err != nil {
		return nil, err
	}

	err = v.plugin.NodePublishVolume(ctx, &csi.NodePublishVolumeRequest{
		VolumeID:          vol.ID,
		PublishContext:    publishContext,
		StagingTargetPath: pluginStagingPath,
		TargetPath:        pluginTargetPath,
		VolumeCapability:  capabilities,
		Readonly:          usage.ReadOnly,
	},
		grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
		grpc_retry.WithMax(3),
		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)),
	)

	return &MountInfo{Source: hostTargetPath}, err
}

// MountVolume performs the steps required for using a given volume
// configuration for the provided allocation.
//
// TODO: Validate remote volume attachment and implement.
func (v *volumeManager) MountVolume(ctx context.Context, vol *structs.CSIVolume, alloc *structs.Allocation, usage *UsageOptions, publishContext map[string]string) (*MountInfo, error) {
	logger := v.logger.With("volume_id", vol.ID, "alloc_id", alloc.ID)
	ctx = hclog.WithContext(ctx, logger)

	if v.requiresStaging {
		if err := v.stageVolume(ctx, vol, usage, publishContext); err != nil {
			return nil, err
		}
	}

	mountInfo, err := v.publishVolume(ctx, vol, alloc, usage, publishContext)
	if err != nil {
		return nil, err
	}

	v.usageTracker.Claim(alloc, vol, usage)

	return mountInfo, nil
}

// unstageVolume is the inverse operation of `stageVolume` and must be called
// once for each staging path that a volume has been staged under.
// It is safe to call multiple times and a plugin is required to return OK if
// the volume has been unstaged or was never staged on the node.
func (v *volumeManager) unstageVolume(ctx context.Context, vol *structs.CSIVolume, usage *UsageOptions) error {
	logger := hclog.FromContext(ctx)
	logger.Trace("Unstaging volume")
	stagingPath := v.stagingDirForVolume(v.containerMountPoint, vol, usage)
	return v.plugin.NodeUnstageVolume(ctx,
		vol.ID,
		stagingPath,
		grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
		grpc_retry.WithMax(3),
		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)),
	)
}

func combineErrors(maybeErrs ...error) error {
	var result *multierror.Error
	for _, err := range maybeErrs {
		if err == nil {
			continue
		}

		result = multierror.Append(result, err)
	}

	return result.ErrorOrNil()
}

func (v *volumeManager) unpublishVolume(ctx context.Context, vol *structs.CSIVolume, alloc *structs.Allocation, usage *UsageOptions) error {
	pluginTargetPath := v.allocDirForVolume(v.containerMountPoint, vol, alloc, usage)

	rpcErr := v.plugin.NodeUnpublishVolume(ctx, vol.ID, pluginTargetPath,
		grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
		grpc_retry.WithMax(3),
		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)),
	)

	hostTargetPath := v.allocDirForVolume(v.mountRoot, vol, alloc, usage)
	if _, err := os.Stat(hostTargetPath); os.IsNotExist(err) {
		// Host Target Path already got destroyed, just return any rpcErr
		return rpcErr
	}

	// Host Target Path was not cleaned up, attempt to do so here. If it's still
	// a mount then removing the dir will fail and we'll return any rpcErr and the
	// file error.
	rmErr := os.Remove(hostTargetPath)
	if rmErr != nil {
		return combineErrors(rpcErr, rmErr)
	}

	// We successfully removed the directory, return any rpcErrors that were
	// encountered, but because we got here, they were probably flaky or was
	// cleaned up externally. We might want to just return `nil` here in the
	// future.
	return rpcErr
}

func (v *volumeManager) UnmountVolume(ctx context.Context, vol *structs.CSIVolume, alloc *structs.Allocation, usage *UsageOptions) error {
	logger := v.logger.With("volume_id", vol.ID, "alloc_id", alloc.ID)
	ctx = hclog.WithContext(ctx, logger)

	err := v.unpublishVolume(ctx, vol, alloc, usage)
	if err != nil {
		return err
	}

	canRelease := v.usageTracker.Free(alloc, vol, usage)
	if !v.requiresStaging || !canRelease {
		return nil
	}

	return v.unstageVolume(ctx, vol, usage)
}
<commit_msg>csimanager/volume: Update MountVolume docstring<commit_after>package csimanager

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"time"

	grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/helper/mount"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/plugins/csi"
)

var _ VolumeMounter = &volumeManager{}

const (
	DefaultMountActionTimeout = 2 * time.Minute
	StagingDirName            = "staging"
	AllocSpecificDirName      = "per-alloc"
)

// volumeManager handles the state of attached volumes for a given CSI Plugin.
//
// volumeManagers outlive the lifetime of a given allocation as volumes may be
// shared by multiple allocations on the same node.
//
// volumes are stored by an enriched volume usage struct as the CSI Spec requires
// slightly different usage based on the given usage model.
type volumeManager struct {
	logger hclog.Logger
	plugin csi.CSIPlugin

	usageTracker *volumeUsageTracker

	// mountRoot is the root of where plugin directories and mounts may be created
	// e.g /opt/nomad.d/statedir/csi/my-csi-plugin/
	mountRoot string

	// containerMountPoint is the location _inside_ the plugin container that the
	// `mountRoot` is bound in to.
	containerMountPoint string

	// requiresStaging shows whether the plugin requires that the volume manager
	// calls NodeStageVolume and NodeUnstageVolume RPCs during setup and teardown
	requiresStaging bool
}

func newVolumeManager(logger hclog.Logger, plugin csi.CSIPlugin, rootDir, containerRootDir string, requiresStaging bool) *volumeManager {
	return &volumeManager{
		logger:              logger.Named("volume_manager"),
		plugin:              plugin,
		mountRoot:           rootDir,
		containerMountPoint: containerRootDir,
		requiresStaging:     requiresStaging,
		usageTracker:        newVolumeUsageTracker(),
	}
}

func (v *volumeManager) stagingDirForVolume(root string, vol *structs.CSIVolume, usage *UsageOptions) string {
	return filepath.Join(root, StagingDirName, vol.ID, usage.ToFS())
}

func (v *volumeManager) allocDirForVolume(root string, vol *structs.CSIVolume, alloc *structs.Allocation, usage *UsageOptions) string {
	return filepath.Join(root, AllocSpecificDirName, alloc.ID, vol.ID, usage.ToFS())
}

// ensureStagingDir attempts to create a directory for use when staging a volume
// and then validates that the path is not already a mount point for e.g an
// existing volume stage.
//
// Returns whether the directory is a pre-existing mountpoint, the staging path,
// and any errors that occurred.
func (v *volumeManager) ensureStagingDir(vol *structs.CSIVolume, usage *UsageOptions) (string, bool, error) {
	stagingPath := v.stagingDirForVolume(v.mountRoot, vol, usage)

	// Make the staging path, owned by the Nomad User
	if err := os.MkdirAll(stagingPath, 0700); err != nil && !os.IsExist(err) {
		return "", false, fmt.Errorf("failed to create staging directory for volume (%s): %v", vol.ID, err)

	}

	// Validate that it is not already a mount point
	m := mount.New()
	isNotMount, err := m.IsNotAMountPoint(stagingPath)
	if err != nil {
		return "", false, fmt.Errorf("mount point detection failed for volume (%s): %v", vol.ID, err)
	}

	return stagingPath, !isNotMount, nil
}

// ensureAllocDir attempts to create a directory for use when publishing a volume
// and then validates that the path is not already a mount point (e.g when reattaching
// to existing allocs).
//
// Returns whether the directory is a pre-existing mountpoint, the publish path,
// and any errors that occurred.
func (v *volumeManager) ensureAllocDir(vol *structs.CSIVolume, alloc *structs.Allocation, usage *UsageOptions) (string, bool, error) {
	allocPath := v.allocDirForVolume(v.mountRoot, vol, alloc, usage)

	// Make the alloc path, owned by the Nomad User
	if err := os.MkdirAll(allocPath, 0700); err != nil && !os.IsExist(err) {
		return "", false, fmt.Errorf("failed to create allocation directory for volume (%s): %v", vol.ID, err)
	}

	// Validate that it is not already a mount point
	m := mount.New()
	isNotMount, err := m.IsNotAMountPoint(allocPath)
	if err != nil {
		return "", false, fmt.Errorf("mount point detection failed for volume (%s): %v", vol.ID, err)
	}

	return allocPath, !isNotMount, nil
}

// stageVolume prepares a volume for use by allocations. When a plugin exposes
// the STAGE_UNSTAGE_VOLUME capability it MUST be called once-per-volume for a
// given usage mode before the volume can be NodePublish-ed.
func (v *volumeManager) stageVolume(ctx context.Context, vol *structs.CSIVolume, usage *UsageOptions, publishContext map[string]string) error {
	logger := hclog.FromContext(ctx)
	logger.Trace("Preparing volume staging environment")
	hostStagingPath, isMount, err := v.ensureStagingDir(vol, usage)
	if err != nil {
		return err
	}
	pluginStagingPath := v.stagingDirForVolume(v.containerMountPoint, vol, usage)

	logger.Trace("Volume staging environment", "pre-existing_mount", isMount, "host_staging_path", hostStagingPath, "plugin_staging_path", pluginStagingPath)

	if isMount {
		logger.Debug("re-using existing staging mount for volume", "staging_path", hostStagingPath)
		return nil
	}

	capability, err := csi.VolumeCapabilityFromStructs(vol.AttachmentMode, vol.AccessMode)
	if err != nil {
		return err
	}

	// We currently treat all explicit CSI NodeStageVolume errors (aside from timeouts, codes.ResourceExhausted, and codes.Unavailable)
	// as fatal.
	// In the future, we can provide more useful error messages based on
	// different types of error. For error documentation see:
	// https://github.com/container-storage-interface/spec/blob/4731db0e0bc53238b93850f43ab05d9355df0fd9/spec.md#nodestagevolume-errors
	return v.plugin.NodeStageVolume(ctx,
		vol.ID,
		publishContext,
		pluginStagingPath,
		capability,
		grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
		grpc_retry.WithMax(3),
		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)),
	)
}

func (v *volumeManager) publishVolume(ctx context.Context, vol *structs.CSIVolume, alloc *structs.Allocation, usage *UsageOptions, publishContext map[string]string) (*MountInfo, error) {
	logger := hclog.FromContext(ctx)
	var pluginStagingPath string
	if v.requiresStaging {
		pluginStagingPath = v.stagingDirForVolume(v.containerMountPoint, vol, usage)
	}

	hostTargetPath, isMount, err := v.ensureAllocDir(vol, alloc, usage)
	if err != nil {
		return nil, err
	}
	pluginTargetPath := v.allocDirForVolume(v.containerMountPoint, vol, alloc, usage)

	if isMount {
		logger.Debug("Re-using existing published volume for allocation")
		return &MountInfo{Source: hostTargetPath}, nil
	}

	capabilities, err := csi.VolumeCapabilityFromStructs(vol.AttachmentMode, vol.AccessMode)
	if err != nil {
		return nil, err
	}

	err = v.plugin.NodePublishVolume(ctx, &csi.NodePublishVolumeRequest{
		VolumeID:          vol.ID,
		PublishContext:    publishContext,
		StagingTargetPath: pluginStagingPath,
		TargetPath:        pluginTargetPath,
		VolumeCapability:  capabilities,
		Readonly:          usage.ReadOnly,
	},
		grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
		grpc_retry.WithMax(3),
		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)),
	)

	return &MountInfo{Source: hostTargetPath}, err
}

// MountVolume performs the steps required for using a given volume
// configuration for the provided allocation.
// It is passed the publishContext from remote attachment, and specific usage
// modes from the CSI Hook.
// It then uses this state to stage and publish the volume as required for use
// by the given allocation.
func (v *volumeManager) MountVolume(ctx context.Context, vol *structs.CSIVolume, alloc *structs.Allocation, usage *UsageOptions, publishContext map[string]string) (*MountInfo, error) {
	logger := v.logger.With("volume_id", vol.ID, "alloc_id", alloc.ID)
	ctx = hclog.WithContext(ctx, logger)

	if v.requiresStaging {
		if err := v.stageVolume(ctx, vol, usage, publishContext); err != nil {
			return nil, err
		}
	}

	mountInfo, err := v.publishVolume(ctx, vol, alloc, usage, publishContext)
	if err != nil {
		return nil, err
	}

	v.usageTracker.Claim(alloc, vol, usage)

	return mountInfo, nil
}

// unstageVolume is the inverse operation of `stageVolume` and must be called
// once for each staging path that a volume has been staged under.
// It is safe to call multiple times and a plugin is required to return OK if
// the volume has been unstaged or was never staged on the node.
func (v *volumeManager) unstageVolume(ctx context.Context, vol *structs.CSIVolume, usage *UsageOptions) error {
	logger := hclog.FromContext(ctx)
	logger.Trace("Unstaging volume")
	stagingPath := v.stagingDirForVolume(v.containerMountPoint, vol, usage)
	return v.plugin.NodeUnstageVolume(ctx,
		vol.ID,
		stagingPath,
		grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
		grpc_retry.WithMax(3),
		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)),
	)
}

func combineErrors(maybeErrs ...error) error {
	var result *multierror.Error
	for _, err := range maybeErrs {
		if err == nil {
			continue
		}

		result = multierror.Append(result, err)
	}

	return result.ErrorOrNil()
}

func (v *volumeManager) unpublishVolume(ctx context.Context, vol *structs.CSIVolume, alloc *structs.Allocation, usage *UsageOptions) error {
	pluginTargetPath := v.allocDirForVolume(v.containerMountPoint, vol, alloc, usage)

	rpcErr := v.plugin.NodeUnpublishVolume(ctx, vol.ID, pluginTargetPath,
		grpc_retry.WithPerRetryTimeout(DefaultMountActionTimeout),
		grpc_retry.WithMax(3),
		grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond)),
	)

	hostTargetPath := v.allocDirForVolume(v.mountRoot, vol, alloc, usage)
	if _, err := os.Stat(hostTargetPath); os.IsNotExist(err) {
		// Host Target Path already got destroyed, just return any rpcErr
		return rpcErr
	}

	// Host Target Path was not cleaned up, attempt to do so here. If it's still
	// a mount then removing the dir will fail and we'll return any rpcErr and the
	// file error.
	rmErr := os.Remove(hostTargetPath)
	if rmErr != nil {
		return combineErrors(rpcErr, rmErr)
	}

	// We successfully removed the directory, return any rpcErrors that were
	// encountered, but because we got here, they were probably flaky or was
	// cleaned up externally. We might want to just return `nil` here in the
	// future.
	return rpcErr
}

func (v *volumeManager) UnmountVolume(ctx context.Context, vol *structs.CSIVolume, alloc *structs.Allocation, usage *UsageOptions) error {
	logger := v.logger.With("volume_id", vol.ID, "alloc_id", alloc.ID)
	ctx = hclog.WithContext(ctx, logger)

	err := v.unpublishVolume(ctx, vol, alloc, usage)
	if err != nil {
		return err
	}

	canRelease := v.usageTracker.Free(alloc, vol, usage)
	if !v.requiresStaging || !canRelease {
		return nil
	}

	return v.unstageVolume(ctx, vol, usage)
}
<|endoftext|>