<commit_before>// +build linux

package fs

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/opencontainers/runc/libcontainer/cgroups"
	"github.com/opencontainers/runc/libcontainer/configs"
	libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils"
	"github.com/pkg/errors"
	"golang.org/x/sys/unix"
)

var (
	subsystemsLegacy = subsystemSet{
		&CpusetGroup{},
		&DevicesGroup{},
		&MemoryGroup{},
		&CpuGroup{},
		&CpuacctGroup{},
		&PidsGroup{},
		&BlkioGroup{},
		&HugetlbGroup{},
		&NetClsGroup{},
		&NetPrioGroup{},
		&PerfEventGroup{},
		&FreezerGroup{},
		&NameGroup{GroupName: "name=systemd", Join: true},
	}
	HugePageSizes, _ = cgroups.GetHugePageSize()
)

var errSubsystemDoesNotExist = errors.New("cgroup: subsystem does not exist")

type subsystemSet []subsystem

func (s subsystemSet) Get(name string) (subsystem, error) {
	for _, ss := range s {
		if ss.Name() == name {
			return ss, nil
		}
	}
	return nil, errSubsystemDoesNotExist
}

type subsystem interface {
	// Name returns the name of the subsystem.
	Name() string
	// Returns the stats, as 'stats', corresponding to the cgroup under 'path'.
	GetStats(path string, stats *cgroups.Stats) error
	// Removes the cgroup represented by 'cgroupData'.
	Remove(*cgroupData) error
	// Creates and joins the cgroup represented by 'cgroupData'.
	Apply(*cgroupData) error
	// Set the cgroup represented by cgroup.
	Set(path string, cgroup *configs.Cgroup) error
}

type manager struct {
	mu       sync.Mutex
	cgroups  *configs.Cgroup
	rootless bool // ignore permission-related errors
	paths    map[string]string
}

func NewManager(cg *configs.Cgroup, paths map[string]string, rootless bool) cgroups.Manager {
	return &manager{
		cgroups:  cg,
		paths:    paths,
		rootless: rootless,
	}
}

// The absolute path to the root of the cgroup hierarchies.
var cgroupRootLock sync.Mutex
var cgroupRoot string

// Gets the cgroupRoot.
func getCgroupRoot() (string, error) {
	cgroupRootLock.Lock()
	defer cgroupRootLock.Unlock()

	if cgroupRoot != "" {
		return cgroupRoot, nil
	}

	f, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return "", err
	}
	defer f.Close()

	var root string
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		text := scanner.Text()
		fields := strings.Split(text, " ")
		// Safe as mountinfo encodes mountpoints with spaces as \040.
		index := strings.Index(text, " - ")
		postSeparatorFields := strings.Fields(text[index+3:])
		numPostFields := len(postSeparatorFields)

		// This is an error as we can't detect if the mount is for "cgroup"
		if numPostFields == 0 {
			return "", fmt.Errorf("mountinfo: found no fields post '-' in %q", text)
		}

		if postSeparatorFields[0] == "cgroup" {
			// Check that the mount is properly formatted.
			if numPostFields < 3 {
				return "", fmt.Errorf("Error found less than 3 fields post '-' in %q", text)
			}

			root = filepath.Dir(fields[4])
			break
		}
	}
	if err := scanner.Err(); err != nil {
		return "", err
	}
	if root == "" {
		return "", errors.New("no cgroup mount found in mountinfo")
	}

	if _, err := os.Stat(root); err != nil {
		return "", err
	}

	cgroupRoot = root
	return cgroupRoot, nil
}

type cgroupData struct {
	root      string
	innerPath string
	config    *configs.Cgroup
	pid       int
}

// isIgnorableError returns whether err is a permission error (in the loose
// sense of the word). This includes EROFS (which for an unprivileged user is
// basically a permission error) and EACCES (for similar reasons) as well as
// the normal EPERM.
func isIgnorableError(rootless bool, err error) bool {
	// We do not ignore errors if we are root.
	if !rootless {
		return false
	}
	// TODO: rm errors.Cause once we switch to %w everywhere
	err = errors.Cause(err)
	// Is it an ordinary EPERM?
	if errors.Is(err, os.ErrPermission) {
		return true
	}
	// Handle some specific syscall errors.
	var errno unix.Errno
	if errors.As(err, &errno) {
		return errno == unix.EROFS || errno == unix.EPERM || errno == unix.EACCES
	}
	return false
}

func (m *manager) getSubsystems() subsystemSet {
	return subsystemsLegacy
}

func (m *manager) Apply(pid int) (err error) {
	if m.cgroups == nil {
		return nil
	}
	m.mu.Lock()
	defer m.mu.Unlock()

	var c = m.cgroups

	d, err := getCgroupData(m.cgroups, pid)
	if err != nil {
		return err
	}

	m.paths = make(map[string]string)
	if c.Paths != nil {
		for name, path := range c.Paths {
			_, err := d.path(name)
			if err != nil {
				if cgroups.IsNotFound(err) {
					continue
				}
				return err
			}
			m.paths[name] = path
		}
		return cgroups.EnterPid(m.paths, pid)
	}

	for _, sys := range m.getSubsystems() {
		p, err := d.path(sys.Name())
		if err != nil {
			// The non-presence of the devices subsystem is
			// considered fatal for security reasons.
			if cgroups.IsNotFound(err) && sys.Name() != "devices" {
				continue
			}
			return err
		}
		m.paths[sys.Name()] = p

		if err := sys.Apply(d); err != nil {
			// In the case of rootless (including euid=0 in userns), where an
			// explicit cgroup path hasn't been set, we don't bail on error in
			// case of permission problems. Cases where limits have been set
			// (and we couldn't create our own cgroup) are handled by Set.
			if isIgnorableError(m.rootless, err) && m.cgroups.Path == "" {
				delete(m.paths, sys.Name())
				continue
			}
			return err
		}

	}
	return nil
}

func (m *manager) Destroy() error {
	if m.cgroups == nil || m.cgroups.Paths != nil {
		return nil
	}
	m.mu.Lock()
	defer m.mu.Unlock()
	if err := cgroups.RemovePaths(m.paths); err != nil {
		return err
	}
	m.paths = make(map[string]string)
	return nil
}

func (m *manager) Path(subsys string) string {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.paths[subsys]
}

func (m *manager) GetStats() (*cgroups.Stats, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	stats := cgroups.NewStats()
	for name, path := range m.paths {
		sys, err := m.getSubsystems().Get(name)
		if err == errSubsystemDoesNotExist || !cgroups.PathExists(path) {
			continue
		}
		if err := sys.GetStats(path, stats); err != nil {
			return nil, err
		}
	}
	return stats, nil
}

func (m *manager) Set(container *configs.Config) error {
	if container.Cgroups == nil {
		return nil
	}

	// If Paths are set, then we are just joining cgroups paths
	// and there is no need to set any values.
	if m.cgroups != nil && m.cgroups.Paths != nil {
		return nil
	}

	m.mu.Lock()
	defer m.mu.Unlock()
	for _, sys := range m.getSubsystems() {
		path := m.paths[sys.Name()]
		if err := sys.Set(path, container.Cgroups); err != nil {
			if m.rootless && sys.Name() == "devices" {
				continue
			}
			// When m.Rootless is true, errors from the device subsystem are ignored because it is really not expected to work.
			// However, errors from other subsystems are not ignored.
			// see @test "runc create (rootless + limits + no cgrouppath + no permission) fails with informative error"
			if path == "" {
				// We never created a path for this cgroup, so we cannot set
				// limits for it (though we have already tried at this point).
				return fmt.Errorf("cannot set %s limit: container could not join or create cgroup", sys.Name())
			}
			return err
		}
	}

	return nil
}

// Freeze toggles the container's freezer cgroup depending on the state
// provided
func (m *manager) Freeze(state configs.FreezerState) (Err error) {
	path := m.Path("freezer")
	if m.cgroups == nil || path == "" {
		return errors.New("cannot toggle freezer: cgroups not configured for container")
	}

	prevState := m.cgroups.Resources.Freezer
	m.cgroups.Resources.Freezer = state
	defer func() {
		if Err != nil {
			m.cgroups.Resources.Freezer = prevState
		}
	}()

	freezer := &FreezerGroup{}
	if err := freezer.Set(path, m.cgroups); err != nil {
		return err
	}
	return nil
}

func (m *manager) GetPids() ([]int, error) {
	return cgroups.GetPids(m.Path("devices"))
}

func (m *manager) GetAllPids() ([]int, error) {
	return cgroups.GetAllPids(m.Path("devices"))
}

func getCgroupData(c *configs.Cgroup, pid int) (*cgroupData, error) {
	root, err := getCgroupRoot()
	if err != nil {
		return nil, err
	}

	if (c.Name != "" || c.Parent != "") && c.Path != "" {
		return nil, errors.New("cgroup: either Path or Name and Parent should be used")
	}

	// XXX: Do not remove this code. Path safety is important! -- cyphar
	cgPath := libcontainerUtils.CleanPath(c.Path)
	cgParent := libcontainerUtils.CleanPath(c.Parent)
	cgName := libcontainerUtils.CleanPath(c.Name)

	innerPath := cgPath
	if innerPath == "" {
		innerPath = filepath.Join(cgParent, cgName)
	}

	return &cgroupData{
		root:      root,
		innerPath: innerPath,
		config:    c,
		pid:       pid,
	}, nil
}

func (raw *cgroupData) path(subsystem string) (string, error) {
	mnt, err := cgroups.FindCgroupMountpoint(raw.root, subsystem)
	// If we didn't mount the subsystem, there is no point we make the path.
	if err != nil {
		return "", err
	}

	// If the cgroup name/path is absolute do not look relative to the cgroup of the init process.
	if filepath.IsAbs(raw.innerPath) {
		// Sometimes subsystems can be mounted together as 'cpu,cpuacct'.
		return filepath.Join(raw.root, filepath.Base(mnt), raw.innerPath), nil
	}

	// Use GetOwnCgroupPath instead of GetInitCgroupPath, because the creating
	// process could in container and shared pid namespace with host, and
	// /proc/1/cgroup could point to whole other world of cgroups.
	parentPath, err := cgroups.GetOwnCgroupPath(subsystem)
	if err != nil {
		return "", err
	}

	return filepath.Join(parentPath, raw.innerPath), nil
}

func (raw *cgroupData) join(subsystem string) (string, error) {
	path, err := raw.path(subsystem)
	if err != nil {
		return "", err
	}
	if err := os.MkdirAll(path, 0755); err != nil {
		return "", err
	}
	if err := cgroups.WriteCgroupProc(path, raw.pid); err != nil {
		return "", err
	}
	return path, nil
}

func removePath(p string, err error) error {
	if err != nil {
		return err
	}
	if p != "" {
		return os.RemoveAll(p)
	}
	return nil
}

func (m *manager) GetPaths() map[string]string {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.paths
}

func (m *manager) GetCgroups() (*configs.Cgroup, error) {
	return m.cgroups, nil
}

func (m *manager) GetFreezerState() (configs.FreezerState, error) {
	dir := m.Path("freezer")
	// If the container doesn't have the freezer cgroup, say it's undefined.
	if dir == "" {
		return configs.Undefined, nil
	}
	freezer := &FreezerGroup{}
	return freezer.GetState(dir)
}

func (m *manager) Exists() bool {
	return cgroups.PathExists(m.Path("devices"))
}
<commit_msg>cgroups/fs/Freeze: simplify<commit_after>// +build linux

package fs

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/opencontainers/runc/libcontainer/cgroups"
	"github.com/opencontainers/runc/libcontainer/configs"
	libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils"
	"github.com/pkg/errors"
	"golang.org/x/sys/unix"
)

var (
	subsystemsLegacy = subsystemSet{
		&CpusetGroup{},
		&DevicesGroup{},
		&MemoryGroup{},
		&CpuGroup{},
		&CpuacctGroup{},
		&PidsGroup{},
		&BlkioGroup{},
		&HugetlbGroup{},
		&NetClsGroup{},
		&NetPrioGroup{},
		&PerfEventGroup{},
		&FreezerGroup{},
		&NameGroup{GroupName: "name=systemd", Join: true},
	}
	HugePageSizes, _ = cgroups.GetHugePageSize()
)

var errSubsystemDoesNotExist = errors.New("cgroup: subsystem does not exist")

type subsystemSet []subsystem

func (s subsystemSet) Get(name string) (subsystem, error) {
	for _, ss := range s {
		if ss.Name() == name {
			return ss, nil
		}
	}
	return nil, errSubsystemDoesNotExist
}

type subsystem interface {
	// Name returns the name of the subsystem.
	Name() string
	// Returns the stats, as 'stats', corresponding to the cgroup under 'path'.
	GetStats(path string, stats *cgroups.Stats) error
	// Removes the cgroup represented by 'cgroupData'.
	Remove(*cgroupData) error
	// Creates and joins the cgroup represented by 'cgroupData'.
	Apply(*cgroupData) error
	// Set the cgroup represented by cgroup.
	Set(path string, cgroup *configs.Cgroup) error
}

type manager struct {
	mu       sync.Mutex
	cgroups  *configs.Cgroup
	rootless bool // ignore permission-related errors
	paths    map[string]string
}

func NewManager(cg *configs.Cgroup, paths map[string]string, rootless bool) cgroups.Manager {
	return &manager{
		cgroups:  cg,
		paths:    paths,
		rootless: rootless,
	}
}

// The absolute path to the root of the cgroup hierarchies.
var cgroupRootLock sync.Mutex
var cgroupRoot string

// Gets the cgroupRoot.
func getCgroupRoot() (string, error) {
	cgroupRootLock.Lock()
	defer cgroupRootLock.Unlock()

	if cgroupRoot != "" {
		return cgroupRoot, nil
	}

	f, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return "", err
	}
	defer f.Close()

	var root string
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		text := scanner.Text()
		fields := strings.Split(text, " ")
		// Safe as mountinfo encodes mountpoints with spaces as \040.
		index := strings.Index(text, " - ")
		postSeparatorFields := strings.Fields(text[index+3:])
		numPostFields := len(postSeparatorFields)

		// This is an error as we can't detect if the mount is for "cgroup"
		if numPostFields == 0 {
			return "", fmt.Errorf("mountinfo: found no fields post '-' in %q", text)
		}

		if postSeparatorFields[0] == "cgroup" {
			// Check that the mount is properly formatted.
			if numPostFields < 3 {
				return "", fmt.Errorf("Error found less than 3 fields post '-' in %q", text)
			}

			root = filepath.Dir(fields[4])
			break
		}
	}
	if err := scanner.Err(); err != nil {
		return "", err
	}
	if root == "" {
		return "", errors.New("no cgroup mount found in mountinfo")
	}

	if _, err := os.Stat(root); err != nil {
		return "", err
	}

	cgroupRoot = root
	return cgroupRoot, nil
}

type cgroupData struct {
	root      string
	innerPath string
	config    *configs.Cgroup
	pid       int
}

// isIgnorableError returns whether err is a permission error (in the loose
// sense of the word). This includes EROFS (which for an unprivileged user is
// basically a permission error) and EACCES (for similar reasons) as well as
// the normal EPERM.
func isIgnorableError(rootless bool, err error) bool {
	// We do not ignore errors if we are root.
	if !rootless {
		return false
	}
	// TODO: rm errors.Cause once we switch to %w everywhere
	err = errors.Cause(err)
	// Is it an ordinary EPERM?
	if errors.Is(err, os.ErrPermission) {
		return true
	}
	// Handle some specific syscall errors.
	var errno unix.Errno
	if errors.As(err, &errno) {
		return errno == unix.EROFS || errno == unix.EPERM || errno == unix.EACCES
	}
	return false
}

func (m *manager) getSubsystems() subsystemSet {
	return subsystemsLegacy
}

func (m *manager) Apply(pid int) (err error) {
	if m.cgroups == nil {
		return nil
	}
	m.mu.Lock()
	defer m.mu.Unlock()

	var c = m.cgroups

	d, err := getCgroupData(m.cgroups, pid)
	if err != nil {
		return err
	}

	m.paths = make(map[string]string)
	if c.Paths != nil {
		for name, path := range c.Paths {
			_, err := d.path(name)
			if err != nil {
				if cgroups.IsNotFound(err) {
					continue
				}
				return err
			}
			m.paths[name] = path
		}
		return cgroups.EnterPid(m.paths, pid)
	}

	for _, sys := range m.getSubsystems() {
		p, err := d.path(sys.Name())
		if err != nil {
			// The non-presence of the devices subsystem is
			// considered fatal for security reasons.
			if cgroups.IsNotFound(err) && sys.Name() != "devices" {
				continue
			}
			return err
		}
		m.paths[sys.Name()] = p

		if err := sys.Apply(d); err != nil {
			// In the case of rootless (including euid=0 in userns), where an
			// explicit cgroup path hasn't been set, we don't bail on error in
			// case of permission problems. Cases where limits have been set
			// (and we couldn't create our own cgroup) are handled by Set.
			if isIgnorableError(m.rootless, err) && m.cgroups.Path == "" {
				delete(m.paths, sys.Name())
				continue
			}
			return err
		}

	}
	return nil
}

func (m *manager) Destroy() error {
	if m.cgroups == nil || m.cgroups.Paths != nil {
		return nil
	}
	m.mu.Lock()
	defer m.mu.Unlock()
	if err := cgroups.RemovePaths(m.paths); err != nil {
		return err
	}
	m.paths = make(map[string]string)
	return nil
}

func (m *manager) Path(subsys string) string {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.paths[subsys]
}

func (m *manager) GetStats() (*cgroups.Stats, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	stats := cgroups.NewStats()
	for name, path := range m.paths {
		sys, err := m.getSubsystems().Get(name)
		if err == errSubsystemDoesNotExist || !cgroups.PathExists(path) {
			continue
		}
		if err := sys.GetStats(path, stats); err != nil {
			return nil, err
		}
	}
	return stats, nil
}

func (m *manager) Set(container *configs.Config) error {
	if container.Cgroups == nil {
		return nil
	}

	// If Paths are set, then we are just joining cgroups paths
	// and there is no need to set any values.
	if m.cgroups != nil && m.cgroups.Paths != nil {
		return nil
	}

	m.mu.Lock()
	defer m.mu.Unlock()
	for _, sys := range m.getSubsystems() {
		path := m.paths[sys.Name()]
		if err := sys.Set(path, container.Cgroups); err != nil {
			if m.rootless && sys.Name() == "devices" {
				continue
			}
			// When m.Rootless is true, errors from the device subsystem are ignored because it is really not expected to work.
			// However, errors from other subsystems are not ignored.
			// see @test "runc create (rootless + limits + no cgrouppath + no permission) fails with informative error"
			if path == "" {
				// We never created a path for this cgroup, so we cannot set
				// limits for it (though we have already tried at this point).
				return fmt.Errorf("cannot set %s limit: container could not join or create cgroup", sys.Name())
			}
			return err
		}
	}

	return nil
}

// Freeze toggles the container's freezer cgroup depending on the state
// provided
func (m *manager) Freeze(state configs.FreezerState) error {
	path := m.Path("freezer")
	if m.cgroups == nil || path == "" {
		return errors.New("cannot toggle freezer: cgroups not configured for container")
	}

	prevState := m.cgroups.Resources.Freezer
	m.cgroups.Resources.Freezer = state
	freezer := &FreezerGroup{}
	if err := freezer.Set(path, m.cgroups); err != nil {
		m.cgroups.Resources.Freezer = prevState
		return err
	}
	return nil
}

func (m *manager) GetPids() ([]int, error) {
	return cgroups.GetPids(m.Path("devices"))
}

func (m *manager) GetAllPids() ([]int, error) {
	return cgroups.GetAllPids(m.Path("devices"))
}

func getCgroupData(c *configs.Cgroup, pid int) (*cgroupData, error) {
	root, err := getCgroupRoot()
	if err != nil {
		return nil, err
	}

	if (c.Name != "" || c.Parent != "") && c.Path != "" {
		return nil, errors.New("cgroup: either Path or Name and Parent should be used")
	}

	// XXX: Do not remove this code. Path safety is important! -- cyphar
	cgPath := libcontainerUtils.CleanPath(c.Path)
	cgParent := libcontainerUtils.CleanPath(c.Parent)
	cgName := libcontainerUtils.CleanPath(c.Name)

	innerPath := cgPath
	if innerPath == "" {
		innerPath = filepath.Join(cgParent, cgName)
	}

	return &cgroupData{
		root:      root,
		innerPath: innerPath,
		config:    c,
		pid:       pid,
	}, nil
}

func (raw *cgroupData) path(subsystem string) (string, error) {
	mnt, err := cgroups.FindCgroupMountpoint(raw.root, subsystem)
	// If we didn't mount the subsystem, there is no point we make the path.
	if err != nil {
		return "", err
	}

	// If the cgroup name/path is absolute do not look relative to the cgroup of the init process.
	if filepath.IsAbs(raw.innerPath) {
		// Sometimes subsystems can be mounted together as 'cpu,cpuacct'.
		return filepath.Join(raw.root, filepath.Base(mnt), raw.innerPath), nil
	}

	// Use GetOwnCgroupPath instead of GetInitCgroupPath, because the creating
	// process could in container and shared pid namespace with host, and
	// /proc/1/cgroup could point to whole other world of cgroups.
	parentPath, err := cgroups.GetOwnCgroupPath(subsystem)
	if err != nil {
		return "", err
	}

	return filepath.Join(parentPath, raw.innerPath), nil
}

func (raw *cgroupData) join(subsystem string) (string, error) {
	path, err := raw.path(subsystem)
	if err != nil {
		return "", err
	}
	if err := os.MkdirAll(path, 0755); err != nil {
		return "", err
	}
	if err := cgroups.WriteCgroupProc(path, raw.pid); err != nil {
		return "", err
	}
	return path, nil
}

func removePath(p string, err error) error {
	if err != nil {
		return err
	}
	if p != "" {
		return os.RemoveAll(p)
	}
	return nil
}

func (m *manager) GetPaths() map[string]string {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.paths
}

func (m *manager) GetCgroups() (*configs.Cgroup, error) {
	return m.cgroups, nil
}

func (m *manager) GetFreezerState() (configs.FreezerState, error) {
	dir := m.Path("freezer")
	// If the container doesn't have the freezer cgroup, say it's undefined.
	if dir == "" {
		return configs.Undefined, nil
	}
	freezer := &FreezerGroup{}
	return freezer.GetState(dir)
}

func (m *manager) Exists() bool {
	return cgroups.PathExists(m.Path("devices"))
}
<|endoftext|>


<commit_before>package sls

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/denverdino/aliyungo/common"
	"github.com/golang/protobuf/proto"
	//"time"
	"os"
	"strconv"
)

type Client struct {
	accessKeyId     string //Access Key Id
	accessKeySecret string //Access Key Secret
	debug           bool
	httpClient      *http.Client
	version         string
	internal        bool
	region          common.Region
	endpoint        string
}

type Project struct {
	client      *Client
	Name        string `json:"projectName,omitempty"`
	Description string `json:"description,omitempty"`
}

//type LogContent struct {
//	Key   string
//	Value string
//}
//
//type LogItem struct {
//	Time     time.Time
//	Contents []*LogContent
//}
//
//type LogGroupItem struct {
//	Logs   []*LogItem
//	Topic  string
//	Source string
//}

type PutLogsRequest struct {
	Project  string
	LogStore string
	LogItems LogGroup
	HashKey  string
}

const (
	SLSDefaultEndpoint = "sls.aliyuncs.com"
	SLSAPIVersion      = "0.6.0"
	METHOD_GET         = "GET"
	METHOD_POST        = "POST"
	METHOD_PUT         = "PUT"
	METHOD_DELETE      = "DELETE"
)

// NewClient creates a new instance of ECS client
func NewClient(region common.Region, internal bool, accessKeyId, accessKeySecret string) *Client {
	endpoint := os.Getenv("SLS_ENDPOINT")
	if endpoint == "" {
		endpoint = SLSDefaultEndpoint
	}
	return NewClientWithEndpoint(endpoint, region, internal, accessKeyId, accessKeySecret)
}

func NewClientWithEndpoint(endpoint string, region common.Region, internal bool, accessKeyId, accessKeySecret string) *Client {
	return &Client{
		accessKeyId:     accessKeyId,
		accessKeySecret: accessKeySecret,
		internal:        internal,
		region:          region,
		version:         SLSAPIVersion,
		endpoint:        endpoint,
		httpClient:      &http.Client{},
	}
}

func (client *Client) Project(name string) (*Project, error) {

	//	newClient := client.forProject(name)
	//
	//	req := &request{
	//		method: METHOD_GET,
	//		path: "/",
	//	}
	//
	//	project := &Project{}
	//
	//	if err := newClient.requestWithJsonResponse(req, project); err != nil {
	//		return nil, err
	//	}
	//	project.client = newClient
	//	return project, nil
	return &Project{
		Name:   name,
		client: client.forProject(name),
	}, nil
}

func (client *Client) forProject(name string) *Client {
	newclient := *client

	region := string(client.region)
	if client.internal {
		region = fmt.Sprintf("%s-intranet", region)
	}
	newclient.endpoint = fmt.Sprintf("%s.%s.%s", name, region, client.endpoint)
	return &newclient
}

func (client *Client) DeleteProject(name string) error {
	req := &request{
		method: METHOD_DELETE,
		path:   "/",
	}

	newClient := client.forProject(name)
	return newClient.requestWithClose(req)
}

func (client *Client) CreateProject(name string, description string) error {
	project := &Project{
		Name:        name,
		Description: description,
	}
	data, err := json.Marshal(project)
	if err != nil {
		return err
	}

	req := &request{
		method:      METHOD_POST,
		path:        "/",
		payload:     data,
		contentType: "application/json",
	}

	newClient := client.forProject(name)
	return newClient.requestWithClose(req)
}

//
//func marshal() ([]byte, error) {
//
//	logGroups := []*LogGroup{}
//	tmp := []*LogGroupItem
//	for _, logGroupItem := range tmp {
//
//		logs := []*Log{}
//		for _, logItem := range logGroupItem.Logs {
//			contents := []*Log_Content{}
//			for key, value := range logItem.Content {
//				contents = append(contents, &Log_Content{
//					Key: proto.String(key),
//					Value: proto.String(value),
//				})
//			}
//
//			logs = append(logs, &Log{
//				Time: proto.Uint32(uint32(LogItem.Time.Unix())),
//				Contents: contents,
//			})
//		}
//
//		logGroup := &LogGroup{
//			Topic: proto.String(LogGroupItem.Topic),
//			Source: proto.String(LogGroupItem.Source),
//			Logs: logs,
//		}
//		logGroups = append(logGroups, logGroup)
//	}
//
//	return proto.Marshal(&LogGroupList{
//		LogGroupList: logGroups,
//	})
//}

func (client *Client) PutLogs(putLogRequest *PutLogsRequest) error {
	if putLogRequest == nil {
		return nil
	}

	data, err := proto.Marshal(&putLogRequest.LogItems)
	if err != nil {
		return err
	}

	req := &request{
		method:      METHOD_POST,
		path:        "/logstores/" + putLogRequest.LogStore + "/shards/lb",
		payload:     data,
		contentType: "application/x-protobuf",
		headers:     map[string]string{"x-log-bodyrawsize": strconv.Itoa(len(data))},
	}

	newClient := client.forProject(putLogRequest.Project)
	return newClient.requestWithClose(req)

}
<commit_msg>修改sls默认endpoint：log.aliyuncs.com<commit_after>package sls

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/denverdino/aliyungo/common"
	"github.com/golang/protobuf/proto"
	//"time"
	"os"
	"strconv"
)

type Client struct {
	accessKeyId     string //Access Key Id
	accessKeySecret string //Access Key Secret
	debug           bool
	httpClient      *http.Client
	version         string
	internal        bool
	region          common.Region
	endpoint        string
}

type Project struct {
	client      *Client
	Name        string `json:"projectName,omitempty"`
	Description string `json:"description,omitempty"`
}

//type LogContent struct {
//	Key   string
//	Value string
//}
//
//type LogItem struct {
//	Time     time.Time
//	Contents []*LogContent
//}
//
//type LogGroupItem struct {
//	Logs   []*LogItem
//	Topic  string
//	Source string
//}

type PutLogsRequest struct {
	Project  string
	LogStore string
	LogItems LogGroup
	HashKey  string
}

const (
	SLSDefaultEndpoint = "log.aliyuncs.com"
	SLSAPIVersion      = "0.6.0"
	METHOD_GET         = "GET"
	METHOD_POST        = "POST"
	METHOD_PUT         = "PUT"
	METHOD_DELETE      = "DELETE"
)

// NewClient creates a new instance of ECS client
func NewClient(region common.Region, internal bool, accessKeyId, accessKeySecret string) *Client {
	endpoint := os.Getenv("SLS_ENDPOINT")
	if endpoint == "" {
		endpoint = SLSDefaultEndpoint
	}
	return NewClientWithEndpoint(endpoint, region, internal, accessKeyId, accessKeySecret)
}

func NewClientWithEndpoint(endpoint string, region common.Region, internal bool, accessKeyId, accessKeySecret string) *Client {
	return &Client{
		accessKeyId:     accessKeyId,
		accessKeySecret: accessKeySecret,
		internal:        internal,
		region:          region,
		version:         SLSAPIVersion,
		endpoint:        endpoint,
		httpClient:      &http.Client{},
	}
}

func (client *Client) Project(name string) (*Project, error) {

	//	newClient := client.forProject(name)
	//
	//	req := &request{
	//		method: METHOD_GET,
	//		path: "/",
	//	}
	//
	//	project := &Project{}
	//
	//	if err := newClient.requestWithJsonResponse(req, project); err != nil {
	//		return nil, err
	//	}
	//	project.client = newClient
	//	return project, nil
	return &Project{
		Name:   name,
		client: client.forProject(name),
	}, nil
}

func (client *Client) forProject(name string) *Client {
	newclient := *client

	region := string(client.region)
	if client.internal {
		region = fmt.Sprintf("%s-intranet", region)
	}
	newclient.endpoint = fmt.Sprintf("%s.%s.%s", name, region, client.endpoint)
	return &newclient
}

func (client *Client) DeleteProject(name string) error {
	req := &request{
		method: METHOD_DELETE,
		path:   "/",
	}

	newClient := client.forProject(name)
	return newClient.requestWithClose(req)
}

func (client *Client) CreateProject(name string, description string) error {
	project := &Project{
		Name:        name,
		Description: description,
	}
	data, err := json.Marshal(project)
	if err != nil {
		return err
	}

	req := &request{
		method:      METHOD_POST,
		path:        "/",
		payload:     data,
		contentType: "application/json",
	}

	newClient := client.forProject(name)
	return newClient.requestWithClose(req)
}

//
//func marshal() ([]byte, error) {
//
//	logGroups := []*LogGroup{}
//	tmp := []*LogGroupItem
//	for _, logGroupItem := range tmp {
//
//		logs := []*Log{}
//		for _, logItem := range logGroupItem.Logs {
//			contents := []*Log_Content{}
//			for key, value := range logItem.Content {
//				contents = append(contents, &Log_Content{
//					Key: proto.String(key),
//					Value: proto.String(value),
//				})
//			}
//
//			logs = append(logs, &Log{
//				Time: proto.Uint32(uint32(LogItem.Time.Unix())),
//				Contents: contents,
//			})
//		}
//
//		logGroup := &LogGroup{
//			Topic: proto.String(LogGroupItem.Topic),
//			Source: proto.String(LogGroupItem.Source),
//			Logs: logs,
//		}
//		logGroups = append(logGroups, logGroup)
//	}
//
//	return proto.Marshal(&LogGroupList{
//		LogGroupList: logGroups,
//	})
//}

func (client *Client) PutLogs(putLogRequest *PutLogsRequest) error {
	if putLogRequest == nil {
		return nil
	}

	data, err := proto.Marshal(&putLogRequest.LogItems)
	if err != nil {
		return err
	}

	req := &request{
		method:      METHOD_POST,
		path:        "/logstores/" + putLogRequest.LogStore + "/shards/lb",
		payload:     data,
		contentType: "application/x-protobuf",
		headers:     map[string]string{"x-log-bodyrawsize": strconv.Itoa(len(data))},
	}

	newClient := client.forProject(putLogRequest.Project)
	return newClient.requestWithClose(req)

}
<|endoftext|>


<commit_before>package runtime

import "testing"

func TestStreamp(t *testing.T) {
	execTests(t, Streamp, []test{
		{
			exp:     `(streamp (standard-input))`,
			want:    `t`,
			wantErr: false,
		},
		{
			exp:     `(streamp '())`,
			want:    `nil`,
			wantErr: false,
		},
	})
}

func TestInputStreamP(t *testing.T) {
	execTests(t, InputStreamP, []test{
		{
			exp:     `(input-stream-p (standard-input))`,
			want:    `t`,
			wantErr: false,
		},
		{
			exp:     `(input-stream-p (standard-output))`,
			want:    `nil`,
			wantErr: false,
		},
		{
			exp:     `(streamp '(a b c))`,
			want:    `nil`,
			wantErr: false,
		},
	})
}

func TestOutputStreamP(t *testing.T) {
	execTests(t, OutputStreamP, []test{
		{
			exp:     `(output-stream-p (standard-input))`,
			want:    `nil`,
			wantErr: false,
		},
		{
			exp:     `(output-stream-p (standard-output))`,
			want:    `t`,
			wantErr: false,
		},
		{
			exp:     `(streamp "hello")`,
			want:    `nil`,
			wantErr: false,
		},
	})
}

func TestWithStandardInput(t *testing.T) {
	execTests(t, WithStandardInput, []test{
		{
			exp: `
			(with-standard-input (create-string-input-stream "this is a string")
 				(list (read) (read)))
			`,
			want:    `'(this is)`,
			wantErr: false,
		},
	})
}

func TestWithOpenIoFile(t *testing.T) {
	execTests(t, WithOpenIoFile, []test{
		{
			exp: `
			(with-open-output-file (outstream "__example.dat")
  				(format outstream "hello"))
			`,
			want:    `nil`,
			wantErr: false,
		},
		{
			exp: `
			(with-open-input-file (instream "__example.dat")
  				(read instream))
			`,
			want:    `'hello`,
			wantErr: false,
		},
	})
}

func TestCreateStringInputStream(t *testing.T) {
	execTests(t, CreateStringInputStream, []test{
		{
			exp: `
			(let ((str (create-string-input-stream "this is a string")))
				(list (read str) (read str) (read str)))
			`,
			want:    `'(this is a)`,
			wantErr: false,
		},
	})
}

func TestCreateStringOutputStream(t *testing.T) {
	execTests(t, CreateStringOutputStream, []test{
		{
			exp: `
			(let ((str (create-string-output-stream)))
			  (format str "hello")
			  (format str "world")
			  (get-output-stream-string str))
			`,
			want:    `"helloworld"`,
			wantErr: false,
		},
	})
}

func TestGetOutputStreamString(t *testing.T) {
	execTests(t, GetOutputStreamString, []test{
		{
			exp: `
			(let ((out-str (create-string-output-stream)))
			  (format out-str "This is a string")
			  (let ((part1 (get-output-stream-string out-str)))
			    (format out-str "right!")
			    (list part1 (get-output-stream-string out-str))))
			`,
			want:    `'("This is a string" "right!")`,
			wantErr: false,
		},
	})
}

func TestRead(t *testing.T) {
	execTests(t, Read, []test{
		{
			exp:     `(defglobal str (create-string-input-stream "hello #(1 2 3) 123 #\\A"))`,
			want:    `'str`,
			wantErr: false,
		},
		{
			exp:     `(read str)`,
			want:    `'hello`,
			wantErr: false,
		},
		{
			exp:     `(read str)`,
			want:    `#(1 2 3)`,
			wantErr: false,
		},
		{
			exp:     `(read str)`,
			want:    `123`,
			wantErr: false,
		},
		{
			exp:     `(read str)`,
			want:    `#\A`,
			wantErr: false,
		},
	})
}

func TestReadChar(t *testing.T) {
	execTests(t, ReadChar, []test{
		{
			exp:     `(defglobal str (create-string-input-stream "hi"))`,
			want:    `'str`,
			wantErr: false,
		},
		{
			exp:     `(read-char str)`,
			want:    `#\h`,
			wantErr: false,
		},
		{
			exp:     `(read-char str)`,
			want:    `#\i`,
			wantErr: false,
		},
		{
			exp:     `(read str)`,
			want:    `nil`,
			wantErr: true,
		},
	})
}

func TestPreviewChar(t *testing.T) {
	execTests(t, PreviewChar, []test{
		{
			exp: `
			(let ((s (create-string-input-stream "foo")))
				(list (preview-char s) (read-char s) (read-char s)))
			`,
			want:    `'(#\f #\f #\o)`,
			wantErr: false,
		},
	})
}
<commit_msg>Add test for read-line<commit_after>package runtime

import "testing"

func TestStreamp(t *testing.T) {
	execTests(t, Streamp, []test{
		{
			exp:     `(streamp (standard-input))`,
			want:    `t`,
			wantErr: false,
		},
		{
			exp:     `(streamp '())`,
			want:    `nil`,
			wantErr: false,
		},
	})
}

func TestInputStreamP(t *testing.T) {
	execTests(t, InputStreamP, []test{
		{
			exp:     `(input-stream-p (standard-input))`,
			want:    `t`,
			wantErr: false,
		},
		{
			exp:     `(input-stream-p (standard-output))`,
			want:    `nil`,
			wantErr: false,
		},
		{
			exp:     `(streamp '(a b c))`,
			want:    `nil`,
			wantErr: false,
		},
	})
}

func TestOutputStreamP(t *testing.T) {
	execTests(t, OutputStreamP, []test{
		{
			exp:     `(output-stream-p (standard-input))`,
			want:    `nil`,
			wantErr: false,
		},
		{
			exp:     `(output-stream-p (standard-output))`,
			want:    `t`,
			wantErr: false,
		},
		{
			exp:     `(streamp "hello")`,
			want:    `nil`,
			wantErr: false,
		},
	})
}

func TestWithStandardInput(t *testing.T) {
	execTests(t, WithStandardInput, []test{
		{
			exp: `
			(with-standard-input (create-string-input-stream "this is a string")
 				(list (read) (read)))
			`,
			want:    `'(this is)`,
			wantErr: false,
		},
	})
}

func TestWithOpenIoFile(t *testing.T) {
	execTests(t, WithOpenIoFile, []test{
		{
			exp: `
			(with-open-output-file (outstream "__example.dat")
  				(format outstream "hello"))
			`,
			want:    `nil`,
			wantErr: false,
		},
		{
			exp: `
			(with-open-input-file (instream "__example.dat")
  				(read instream))
			`,
			want:    `'hello`,
			wantErr: false,
		},
	})
}

func TestCreateStringInputStream(t *testing.T) {
	execTests(t, CreateStringInputStream, []test{
		{
			exp: `
			(let ((str (create-string-input-stream "this is a string")))
				(list (read str) (read str) (read str)))
			`,
			want:    `'(this is a)`,
			wantErr: false,
		},
	})
}

func TestCreateStringOutputStream(t *testing.T) {
	execTests(t, CreateStringOutputStream, []test{
		{
			exp: `
			(let ((str (create-string-output-stream)))
			  (format str "hello")
			  (format str "world")
			  (get-output-stream-string str))
			`,
			want:    `"helloworld"`,
			wantErr: false,
		},
	})
}

func TestGetOutputStreamString(t *testing.T) {
	execTests(t, GetOutputStreamString, []test{
		{
			exp: `
			(let ((out-str (create-string-output-stream)))
			  (format out-str "This is a string")
			  (let ((part1 (get-output-stream-string out-str)))
			    (format out-str "right!")
			    (list part1 (get-output-stream-string out-str))))
			`,
			want:    `'("This is a string" "right!")`,
			wantErr: false,
		},
	})
}

func TestRead(t *testing.T) {
	execTests(t, Read, []test{
		{
			exp:     `(defglobal str (create-string-input-stream "hello #(1 2 3) 123 #\\A"))`,
			want:    `'str`,
			wantErr: false,
		},
		{
			exp:     `(read str)`,
			want:    `'hello`,
			wantErr: false,
		},
		{
			exp:     `(read str)`,
			want:    `#(1 2 3)`,
			wantErr: false,
		},
		{
			exp:     `(read str)`,
			want:    `123`,
			wantErr: false,
		},
		{
			exp:     `(read str)`,
			want:    `#\A`,
			wantErr: false,
		},
	})
}

func TestReadChar(t *testing.T) {
	execTests(t, ReadChar, []test{
		{
			exp:     `(defglobal str (create-string-input-stream "hi"))`,
			want:    `'str`,
			wantErr: false,
		},
		{
			exp:     `(read-char str)`,
			want:    `#\h`,
			wantErr: false,
		},
		{
			exp:     `(read-char str)`,
			want:    `#\i`,
			wantErr: false,
		},
		{
			exp:     `(read str)`,
			want:    `nil`,
			wantErr: true,
		},
	})
}

func TestPreviewChar(t *testing.T) {
	execTests(t, PreviewChar, []test{
		{
			exp: `
			(let ((s (create-string-input-stream "foo")))
				(list (preview-char s) (read-char s) (read-char s)))
			`,
			want:    `'(#\f #\f #\o)`,
			wantErr: false,
		},
	})
}

func TestReadLine(t *testing.T) {
	execTests(t, ReadLine, []test{
		{
			exp: `
			(with-open-output-file (out "__newfile")
			  (format out "This is an example")
			  (format out "~%")
			  (format out "look at the output file"))
			`,
			want:    `nil`,
			wantErr: false,
		},
		{
			exp: `
			(defglobal str (open-input-file "__newfile"))
			`,
			want:    `'str`,
			wantErr: false,
		},
		{
			exp: `
			(read-line str)
			`,
			want:    `"This is an example"`,
			wantErr: false,
		},
		{
			exp: `
			(read-line str)
			`,
			want:    `"look at the output file"`,
			wantErr: false,
		},
	})
}
<|endoftext|>


<commit_before>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"encoding/json"
	"fmt"
	"regexp"
	"sort"
	"strings"
)

const (
	// ExportedLabelPrefix is the prefix to prepend to the label names present in
	// exported metrics if a label of the same name is added by the server.
	ExportedLabelPrefix = "exported_"

	// MetricNameLabel is the label name indicating the metric name of a
	// timeseries.
	MetricNameLabel = "__name__"

	// SchemeLabel is the name of the label that holds the scheme on which to
	// scrape a target.
	SchemeLabel = "__scheme__"

	// AddressLabel is the name of the label that holds the address of
	// a scrape target.
	AddressLabel = "__address__"

	// MetricsPathLabel is the name of the label that holds the path on which to
	// scrape a target.
	MetricsPathLabel = "__metrics_path__"

	// ReservedLabelPrefix is a prefix which is not legal in user-supplied
	// label names.
	ReservedLabelPrefix = "__"

	// MetaLabelPrefix is a prefix for labels that provide meta information.
	// Labels with this prefix are used for intermediate label processing and
	// will not be attached to time series.
	MetaLabelPrefix = "__meta_"

	// TmpLabelPrefix is a prefix for temporary labels as part of relabelling.
	// Labels with this prefix are used for intermediate label processing and
	// will not be attached to time series. This is reserved for use in
	// Prometheus configuration files by users.
	TmpLabelPrefix = "__tmp_"

	// ParamLabelPrefix is a prefix for labels that provide URL parameters
	// used to scrape a target.
	ParamLabelPrefix = "__param_"

	// JobLabel is the label name indicating the job from which a timeseries
	// was scraped.
	JobLabel = "job"

	// InstanceLabel is the label name used for the instance label.
	InstanceLabel = "instance"

	// BucketLabel is used for the label that defines the upper bound of a
	// bucket of a histogram ("le" -> "less or equal").
	BucketLabel = "le"

	// QuantileLabel is used for the label that defines the quantile in a
	// summary.
	QuantileLabel = "quantile"
)

// LabelNameRE is a regular expression matching valid label names.
var LabelNameRE = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$")

// A LabelName is a key for a LabelSet or Metric.  It has a value associated
// therewith.
type LabelName string

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (ln *LabelName) UnmarshalYAML(unmarshal func(interface{}) error) error {
	var s string
	if err := unmarshal(&s); err != nil {
		return err
	}
	if !LabelNameRE.MatchString(s) {
		return fmt.Errorf("%q is not a valid label name", s)
	}
	*ln = LabelName(s)
	return nil
}

// UnmarshalJSON implements the json.Unmarshaler interface.
func (ln *LabelName) UnmarshalJSON(b []byte) error {
	var s string
	if err := json.Unmarshal(b, &s); err != nil {
		return err
	}
	if !LabelNameRE.MatchString(s) {
		return fmt.Errorf("%q is not a valid label name", s)
	}
	*ln = LabelName(s)
	return nil
}

// LabelNames is a sortable LabelName slice. In implements sort.Interface.
type LabelNames []LabelName

func (l LabelNames) Len() int {
	return len(l)
}

func (l LabelNames) Less(i, j int) bool {
	return l[i] < l[j]
}

func (l LabelNames) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}

func (l LabelNames) String() string {
	labelStrings := make([]string, 0, len(l))
	for _, label := range l {
		labelStrings = append(labelStrings, string(label))
	}
	return strings.Join(labelStrings, ", ")
}

// A LabelValue is an associated value for a LabelName.
type LabelValue string

// LabelValues is a sortable LabelValue slice. It implements sort.Interface.
type LabelValues []LabelValue

func (l LabelValues) Len() int {
	return len(l)
}

func (l LabelValues) Less(i, j int) bool {
	return sort.StringsAreSorted([]string{string(l[i]), string(l[j])})
}

func (l LabelValues) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}

// LabelPair pairs a name with a value.
type LabelPair struct {
	Name  LabelName
	Value LabelValue
}

// LabelPairs is a sortable slice of LabelPair pointers. It implements
// sort.Interface.
type LabelPairs []*LabelPair

func (l LabelPairs) Len() int {
	return len(l)
}

func (l LabelPairs) Less(i, j int) bool {
	switch {
	case l[i].Name > l[j].Name:
		return false
	case l[i].Name < l[j].Name:
		return true
	case l[i].Value > l[j].Value:
		return false
	case l[i].Value < l[j].Value:
		return true
	default:
		return false
	}
}

func (l LabelPairs) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}
<commit_msg>Add alertname label constant<commit_after>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"encoding/json"
	"fmt"
	"regexp"
	"sort"
	"strings"
)

const (
	// AlertNameLabel is the name of the label containing the an alert's name.
	AlertNameLabel = "alertname"

	// ExportedLabelPrefix is the prefix to prepend to the label names present in
	// exported metrics if a label of the same name is added by the server.
	ExportedLabelPrefix = "exported_"

	// MetricNameLabel is the label name indicating the metric name of a
	// timeseries.
	MetricNameLabel = "__name__"

	// SchemeLabel is the name of the label that holds the scheme on which to
	// scrape a target.
	SchemeLabel = "__scheme__"

	// AddressLabel is the name of the label that holds the address of
	// a scrape target.
	AddressLabel = "__address__"

	// MetricsPathLabel is the name of the label that holds the path on which to
	// scrape a target.
	MetricsPathLabel = "__metrics_path__"

	// ReservedLabelPrefix is a prefix which is not legal in user-supplied
	// label names.
	ReservedLabelPrefix = "__"

	// MetaLabelPrefix is a prefix for labels that provide meta information.
	// Labels with this prefix are used for intermediate label processing and
	// will not be attached to time series.
	MetaLabelPrefix = "__meta_"

	// TmpLabelPrefix is a prefix for temporary labels as part of relabelling.
	// Labels with this prefix are used for intermediate label processing and
	// will not be attached to time series. This is reserved for use in
	// Prometheus configuration files by users.
	TmpLabelPrefix = "__tmp_"

	// ParamLabelPrefix is a prefix for labels that provide URL parameters
	// used to scrape a target.
	ParamLabelPrefix = "__param_"

	// JobLabel is the label name indicating the job from which a timeseries
	// was scraped.
	JobLabel = "job"

	// InstanceLabel is the label name used for the instance label.
	InstanceLabel = "instance"

	// BucketLabel is used for the label that defines the upper bound of a
	// bucket of a histogram ("le" -> "less or equal").
	BucketLabel = "le"

	// QuantileLabel is used for the label that defines the quantile in a
	// summary.
	QuantileLabel = "quantile"
)

// LabelNameRE is a regular expression matching valid label names.
var LabelNameRE = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$")

// A LabelName is a key for a LabelSet or Metric.  It has a value associated
// therewith.
type LabelName string

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (ln *LabelName) UnmarshalYAML(unmarshal func(interface{}) error) error {
	var s string
	if err := unmarshal(&s); err != nil {
		return err
	}
	if !LabelNameRE.MatchString(s) {
		return fmt.Errorf("%q is not a valid label name", s)
	}
	*ln = LabelName(s)
	return nil
}

// UnmarshalJSON implements the json.Unmarshaler interface.
func (ln *LabelName) UnmarshalJSON(b []byte) error {
	var s string
	if err := json.Unmarshal(b, &s); err != nil {
		return err
	}
	if !LabelNameRE.MatchString(s) {
		return fmt.Errorf("%q is not a valid label name", s)
	}
	*ln = LabelName(s)
	return nil
}

// LabelNames is a sortable LabelName slice. In implements sort.Interface.
type LabelNames []LabelName

func (l LabelNames) Len() int {
	return len(l)
}

func (l LabelNames) Less(i, j int) bool {
	return l[i] < l[j]
}

func (l LabelNames) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}

func (l LabelNames) String() string {
	labelStrings := make([]string, 0, len(l))
	for _, label := range l {
		labelStrings = append(labelStrings, string(label))
	}
	return strings.Join(labelStrings, ", ")
}

// A LabelValue is an associated value for a LabelName.
type LabelValue string

// LabelValues is a sortable LabelValue slice. It implements sort.Interface.
type LabelValues []LabelValue

func (l LabelValues) Len() int {
	return len(l)
}

func (l LabelValues) Less(i, j int) bool {
	return sort.StringsAreSorted([]string{string(l[i]), string(l[j])})
}

func (l LabelValues) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}

// LabelPair pairs a name with a value.
type LabelPair struct {
	Name  LabelName
	Value LabelValue
}

// LabelPairs is a sortable slice of LabelPair pointers. It implements
// sort.Interface.
type LabelPairs []*LabelPair

func (l LabelPairs) Len() int {
	return len(l)
}

func (l LabelPairs) Less(i, j int) bool {
	switch {
	case l[i].Name > l[j].Name:
		return false
	case l[i].Name < l[j].Name:
		return true
	case l[i].Value > l[j].Value:
		return false
	case l[i].Value < l[j].Value:
		return true
	default:
		return false
	}
}

func (l LabelPairs) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package addons

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"

	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"k8s.io/minikube/pkg/minikube/assets"
	"k8s.io/minikube/pkg/minikube/cluster"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/machine"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/storageclass"
	pkgutil "k8s.io/minikube/pkg/util"
)

// defaultStorageClassProvisioner is the name of the default storage class provisioner
const defaultStorageClassProvisioner = "standard"

// Set sets a value
func Set(name, value, profile string) error {
	a, valid := isAddonValid(name)
	if !valid {
		return errors.Errorf("%s is not a valid addon", name)
	}

	// Run any additional validations for this property
	if err := run(name, value, profile, a.validations); err != nil {
		return errors.Wrap(err, "running validations")
	}

	// Set the value
	c, err := config.Load(profile)
	if err != nil {
		return errors.Wrap(err, "loading profile")
	}

	if err := a.set(c, name, value); err != nil {
		return errors.Wrap(err, "setting new value of addon")
	}

	// Run any callbacks for this property
	if err := run(name, value, profile, a.callbacks); err != nil {
		return errors.Wrap(err, "running callbacks")
	}

	// Write the value
	return config.Write(profile, c)
}

// Runs all the validation or callback functions and collects errors
func run(name, value, profile string, fns []setFn) error {
	var errors []error
	for _, fn := range fns {
		err := fn(name, value, profile)
		if err != nil {
			errors = append(errors, err)
		}
	}
	if len(errors) > 0 {
		return fmt.Errorf("%v", errors)
	}
	return nil
}

// SetBool sets a bool value
func SetBool(m *config.MachineConfig, name string, val string) error {
	b, err := strconv.ParseBool(val)
	if err != nil {
		return err
	}
	if m.Addons == nil {
		m.Addons = map[string]bool{}
	}
	m.Addons[name] = b
	return nil
}

// enableOrDisableAddon updates addon status executing any commands necessary
func enableOrDisableAddon(name, val, profile string) error {
	enable, err := strconv.ParseBool(val)
	if err != nil {
		return errors.Wrapf(err, "parsing bool: %s", name)
	}
	addon := assets.Addons[name]

	// check addon status before enabling/disabling it
	alreadySet, err := isAddonAlreadySet(addon, enable)
	if err != nil {
		out.ErrT(out.Conflict, "{{.error}}", out.V{"error": err})
		return err
	}
	//if addon is already enabled or disabled, do nothing
	if alreadySet {
		return nil
	}

	if name == "istio" && enable {
		minMem := 8192
		minCpus := 4
		memorySizeMB := pkgutil.CalculateSizeInMB(viper.GetString("memory"))
		cpuCount := viper.GetInt("cpus")
		if memorySizeMB < minMem || cpuCount < minCpus {
			out.WarningT("Enable istio needs {{.minMem}} MB of memory and {{.minCpus}} CPUs.", out.V{"minMem": minMem, "minCpus": minCpus})
		}
	}

	// TODO(r2d4): config package should not reference API, pull this out
	api, err := machine.NewAPIClient()
	if err != nil {
		return errors.Wrap(err, "machine client")
	}
	defer api.Close()

	//if minikube is not running, we return and simply update the value in the addon
	//config and rewrite the file
	if !cluster.IsMinikubeRunning(api) {
		return nil
	}

	cfg, err := config.Load(profile)
	if err != nil && !os.IsNotExist(err) {
		exit.WithCodeT(exit.Data, "Unable to load config: {{.error}}", out.V{"error": err})
	}

	host, err := cluster.CheckIfHostExistsAndLoad(api, cfg.Name)
	if err != nil {
		return errors.Wrap(err, "getting host")
	}

	cmd, err := machine.CommandRunner(host)
	if err != nil {
		return errors.Wrap(err, "command runner")
	}

	data := assets.GenerateTemplateData(cfg.KubernetesConfig)
	return enableOrDisableAddonInternal(addon, cmd, data, enable, profile)
}

func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
	addonStatus, err := addon.IsEnabled()

	if err != nil {
		return false, errors.Wrap(err, "get the addon status")
	}

	if addonStatus && enable {
		return true, nil
	} else if !addonStatus && !enable {
		return true, nil
	}

	return false, nil
}

func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool, profile string) error {
	files := []string{}
	for _, addon := range addon.Assets {
		var addonFile assets.CopyableFile
		var err error
		if addon.IsTemplate() {
			addonFile, err = addon.Evaluate(data)
			if err != nil {
				return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName())
			}

		} else {
			addonFile = addon
		}
		if enable {
			if err := cmd.Copy(addonFile); err != nil {
				return err
			}
		} else {
			defer cmd.Remove(addonFile)
		}
		files = append(files, filepath.Join(addonFile.GetTargetDir(), addonFile.GetTargetName()))
	}
	command, err := kubectlCommand(profile, files, enable)
	if err != nil {
		return err
	}
	if result, err := cmd.RunCmd(command); err != nil {
		return errors.Wrapf(err, "error enabling addon:\n%s", result.Output())
	}
	return nil
}

// enableOrDisableStorageClasses enables or disables storage classes
func enableOrDisableStorageClasses(name, val, profile string) error {
	enable, err := strconv.ParseBool(val)
	if err != nil {
		return errors.Wrap(err, "Error parsing boolean")
	}

	class := defaultStorageClassProvisioner
	if name == "storage-provisioner-gluster" {
		class = "glusterfile"
	}
	storagev1, err := storageclass.GetStoragev1()
	if err != nil {
		return errors.Wrapf(err, "Error getting storagev1 interface %v ", err)
	}

	if enable {
		// Only StorageClass for 'name' should be marked as default
		err = storageclass.SetDefaultStorageClass(storagev1, class)
		if err != nil {
			return errors.Wrapf(err, "Error making %s the default storage class", class)
		}
	} else {
		// Unset the StorageClass as default
		err := storageclass.DisableDefaultStorageClass(storagev1, class)
		if err != nil {
			return errors.Wrapf(err, "Error disabling %s as the default storage class", class)
		}
	}

	return enableOrDisableAddon(name, val, profile)
}
<commit_msg>check cmd.Remove error for linting<commit_after>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package addons

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"k8s.io/minikube/pkg/minikube/assets"
	"k8s.io/minikube/pkg/minikube/cluster"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/machine"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/storageclass"
	pkgutil "k8s.io/minikube/pkg/util"
)

// defaultStorageClassProvisioner is the name of the default storage class provisioner
const defaultStorageClassProvisioner = "standard"

// Set sets a value
func Set(name, value, profile string) error {
	a, valid := isAddonValid(name)
	if !valid {
		return errors.Errorf("%s is not a valid addon", name)
	}

	// Run any additional validations for this property
	if err := run(name, value, profile, a.validations); err != nil {
		return errors.Wrap(err, "running validations")
	}

	// Set the value
	c, err := config.Load(profile)
	if err != nil {
		return errors.Wrap(err, "loading profile")
	}

	if err := a.set(c, name, value); err != nil {
		return errors.Wrap(err, "setting new value of addon")
	}

	// Run any callbacks for this property
	if err := run(name, value, profile, a.callbacks); err != nil {
		return errors.Wrap(err, "running callbacks")
	}

	// Write the value
	return config.Write(profile, c)
}

// Runs all the validation or callback functions and collects errors
func run(name, value, profile string, fns []setFn) error {
	var errors []error
	for _, fn := range fns {
		err := fn(name, value, profile)
		if err != nil {
			errors = append(errors, err)
		}
	}
	if len(errors) > 0 {
		return fmt.Errorf("%v", errors)
	}
	return nil
}

// SetBool sets a bool value
func SetBool(m *config.MachineConfig, name string, val string) error {
	b, err := strconv.ParseBool(val)
	if err != nil {
		return err
	}
	if m.Addons == nil {
		m.Addons = map[string]bool{}
	}
	m.Addons[name] = b
	return nil
}

// enableOrDisableAddon updates addon status executing any commands necessary
func enableOrDisableAddon(name, val, profile string) error {
	enable, err := strconv.ParseBool(val)
	if err != nil {
		return errors.Wrapf(err, "parsing bool: %s", name)
	}
	addon := assets.Addons[name]

	// check addon status before enabling/disabling it
	alreadySet, err := isAddonAlreadySet(addon, enable)
	if err != nil {
		out.ErrT(out.Conflict, "{{.error}}", out.V{"error": err})
		return err
	}
	//if addon is already enabled or disabled, do nothing
	if alreadySet {
		return nil
	}

	if name == "istio" && enable {
		minMem := 8192
		minCpus := 4
		memorySizeMB := pkgutil.CalculateSizeInMB(viper.GetString("memory"))
		cpuCount := viper.GetInt("cpus")
		if memorySizeMB < minMem || cpuCount < minCpus {
			out.WarningT("Enable istio needs {{.minMem}} MB of memory and {{.minCpus}} CPUs.", out.V{"minMem": minMem, "minCpus": minCpus})
		}
	}

	// TODO(r2d4): config package should not reference API, pull this out
	api, err := machine.NewAPIClient()
	if err != nil {
		return errors.Wrap(err, "machine client")
	}
	defer api.Close()

	//if minikube is not running, we return and simply update the value in the addon
	//config and rewrite the file
	if !cluster.IsMinikubeRunning(api) {
		return nil
	}

	cfg, err := config.Load(profile)
	if err != nil && !os.IsNotExist(err) {
		exit.WithCodeT(exit.Data, "Unable to load config: {{.error}}", out.V{"error": err})
	}

	host, err := cluster.CheckIfHostExistsAndLoad(api, cfg.Name)
	if err != nil {
		return errors.Wrap(err, "getting host")
	}

	cmd, err := machine.CommandRunner(host)
	if err != nil {
		return errors.Wrap(err, "command runner")
	}

	data := assets.GenerateTemplateData(cfg.KubernetesConfig)
	return enableOrDisableAddonInternal(addon, cmd, data, enable, profile)
}

func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
	addonStatus, err := addon.IsEnabled()

	if err != nil {
		return false, errors.Wrap(err, "get the addon status")
	}

	if addonStatus && enable {
		return true, nil
	} else if !addonStatus && !enable {
		return true, nil
	}

	return false, nil
}

func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool, profile string) error {
	files := []string{}
	for _, addon := range addon.Assets {
		var addonFile assets.CopyableFile
		var err error
		if addon.IsTemplate() {
			addonFile, err = addon.Evaluate(data)
			if err != nil {
				return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName())
			}

		} else {
			addonFile = addon
		}
		if enable {
			if err := cmd.Copy(addonFile); err != nil {
				return err
			}
		} else {
			defer func() {
				if err := cmd.Remove(addonFile); err != nil {
					glog.Warningf("error removing %s; addon should still be disabled as expected", addonFile)
				}
			}()
		}
		files = append(files, filepath.Join(addonFile.GetTargetDir(), addonFile.GetTargetName()))
	}
	command, err := kubectlCommand(profile, files, enable)
	if err != nil {
		return err
	}
	if result, err := cmd.RunCmd(command); err != nil {
		return errors.Wrapf(err, "error updating addon:\n%s", result.Output())
	}
	return nil
}

// enableOrDisableStorageClasses enables or disables storage classes
func enableOrDisableStorageClasses(name, val, profile string) error {
	enable, err := strconv.ParseBool(val)
	if err != nil {
		return errors.Wrap(err, "Error parsing boolean")
	}

	class := defaultStorageClassProvisioner
	if name == "storage-provisioner-gluster" {
		class = "glusterfile"
	}
	storagev1, err := storageclass.GetStoragev1()
	if err != nil {
		return errors.Wrapf(err, "Error getting storagev1 interface %v ", err)
	}

	if enable {
		// Only StorageClass for 'name' should be marked as default
		err = storageclass.SetDefaultStorageClass(storagev1, class)
		if err != nil {
			return errors.Wrapf(err, "Error making %s the default storage class", class)
		}
	} else {
		// Unset the StorageClass as default
		err := storageclass.DisableDefaultStorageClass(storagev1, class)
		if err != nil {
			return errors.Wrapf(err, "Error disabling %s as the default storage class", class)
		}
	}

	return enableOrDisableAddon(name, val, profile)
}
<|endoftext|>


<commit_before>package chunkenc

import (
	"bufio"
	"bytes"
	"io"
	"sync"

	"github.com/golang/snappy"
	"github.com/klauspost/compress/gzip"
	"github.com/pierrec/lz4"
	"github.com/prometheus/prometheus/pkg/pool"
)

// WriterPool is a pool of io.Writer
// This is used by every chunk to avoid unnecessary allocations.
type WriterPool interface {
	GetWriter(io.Writer) io.WriteCloser
	PutWriter(io.WriteCloser)
}

// ReaderPool similar to WriterPool but for reading chunks.
type ReaderPool interface {
	GetReader(io.Reader) io.Reader
	PutReader(io.Reader)
}

var (
	// Gzip is the gnu zip compression pool
	Gzip = GzipPool{level: gzip.DefaultCompression}
	// LZ4 is the l4z compression pool
	Lz4_64k  = LZ4Pool{bufferSize: 1 << 16}
	Lz4_256k = LZ4Pool{bufferSize: 1 << 18}
	Lz4_1M   = LZ4Pool{bufferSize: 1 << 20}
	Lz4_4M   = LZ4Pool{bufferSize: 1 << 22}

	// Snappy is the snappy compression pool
	Snappy SnappyPool
	// Noop is the no compression pool
	Noop NoopPool

	// BufReaderPool is bufio.Reader pool
	BufReaderPool = &BufioReaderPool{
		pool: sync.Pool{
			New: func() interface{} { return bufio.NewReader(nil) },
		},
	}
	// BytesBufferPool is a bytes buffer used for lines decompressed.
	// Buckets [0.5KB,1KB,2KB,4KB,8KB]
	BytesBufferPool          = pool.New(1<<9, 1<<13, 2, func(size int) interface{} { return make([]byte, 0, size) })
	serializeBytesBufferPool = sync.Pool{
		New: func() interface{} {
			return &bytes.Buffer{}
		},
	}
)

func getWriterPool(enc Encoding) WriterPool {
	return getReaderPool(enc).(WriterPool)
}

func getReaderPool(enc Encoding) ReaderPool {
	switch enc {
	case EncGZIP:
		return &Gzip
	case EncLZ4_64k:
		return &Lz4_64k
	case EncLZ4_256k:
		return &Lz4_256k
	case EncLZ4_1M:
		return &Lz4_1M
	case EncLZ4_4M:
		return &Lz4_4M
	case EncSnappy:
		return &Snappy
	case EncNone:
		return &Noop
	default:
		panic("unknown encoding")
	}
}

// GzipPool is a gun zip compression pool
type GzipPool struct {
	readers sync.Pool
	writers sync.Pool
	level   int
}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *GzipPool) GetReader(src io.Reader) io.Reader {
	if r := pool.readers.Get(); r != nil {
		reader := r.(*gzip.Reader)
		err := reader.Reset(src)
		if err != nil {
			panic(err)
		}
		return reader
	}
	reader, err := gzip.NewReader(src)
	if err != nil {
		panic(err)
	}
	return reader
}

// PutReader places back in the pool a CompressionReader
func (pool *GzipPool) PutReader(reader io.Reader) {
	pool.readers.Put(reader)
}

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *GzipPool) GetWriter(dst io.Writer) io.WriteCloser {
	if w := pool.writers.Get(); w != nil {
		writer := w.(*gzip.Writer)
		writer.Reset(dst)
		return writer
	}

	level := pool.level
	if level == 0 {
		level = gzip.DefaultCompression
	}
	w, err := gzip.NewWriterLevel(dst, level)
	if err != nil {
		panic(err) // never happens, error is only returned on wrong compression level.
	}
	return w
}

// PutWriter places back in the pool a CompressionWriter
func (pool *GzipPool) PutWriter(writer io.WriteCloser) {
	pool.writers.Put(writer)
}

type LZ4Pool struct {
	readers    sync.Pool
	writers    sync.Pool
	bufferSize int // available values: 1<<16 (64k), 1<<18 (256k), 1<<20 (1M), 1<<22 (4M). Defaults to 4MB, if not set.
}

// lz4Reader is simple wrapper around *lz4.Reader, which remembers max used block size,
// as reported by this reader. It is used to determine whether we want to reuse it,
// or throw away and garbage-collect.
type lz4Reader struct {
	r            *lz4.Reader
	maxBlockSize int
}

func (l *lz4Reader) Read(p []byte) (n int, err error) {
	return l.r.Read(p)
}

func (l *lz4Reader) Reset(src io.Reader) {
	l.r.Reset(src)
}

func (l *lz4Reader) onBlockDone(_ int) {
	// remember max block size used.
	if l.r.BlockMaxSize > l.maxBlockSize {
		l.maxBlockSize = l.r.BlockMaxSize
	}
}

func newLz4Reader(src io.Reader) *lz4Reader {
	lz4r := lz4.NewReader(src)
	r := &lz4Reader{r: lz4r}
	lz4r.OnBlockDone = r.onBlockDone
	return r
}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *LZ4Pool) GetReader(src io.Reader) io.Reader {
	if r := pool.readers.Get(); r != nil {
		reader := r.(*lz4Reader)
		reader.Reset(src)
		return reader
	}
	// no need to set buffer size here. Reader uses buffer size based on
	// LZ4 header that it is reading.
	return newLz4Reader(src)
}

// PutReader places back in the pool a CompressionReader
func (pool *LZ4Pool) PutReader(reader io.Reader) {
	r := reader.(*lz4Reader)
	if r.maxBlockSize > pool.bufferSize {
		// Readers base their buffer size based on headers from LZ4 stream.
		// If this reader uses bigger buffer than what we use currently, don't pool it.
		// Reading from a couple of chunks that used big buffer sizes could otherwise quickly lead
		// to high pooled memory usage.
		return
	}
	pool.readers.Put(reader)
}

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *LZ4Pool) GetWriter(dst io.Writer) io.WriteCloser {
	if w := pool.writers.Get(); w != nil {
		writer := w.(*lz4.Writer)
		writer.Reset(dst)
		return writer
	}
	w := lz4.NewWriter(dst)
	w.BlockMaxSize = pool.bufferSize
	return w
}

// PutWriter places back in the pool a CompressionWriter
func (pool *LZ4Pool) PutWriter(writer io.WriteCloser) {
	pool.writers.Put(writer)
}

type SnappyPool struct {
	readers sync.Pool
	writers sync.Pool
}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *SnappyPool) GetReader(src io.Reader) io.Reader {
	if r := pool.readers.Get(); r != nil {
		reader := r.(*snappy.Reader)
		reader.Reset(src)
		return reader
	}
	return snappy.NewReader(src)
}

// PutReader places back in the pool a CompressionReader
func (pool *SnappyPool) PutReader(reader io.Reader) {
	pool.readers.Put(reader)
}

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *SnappyPool) GetWriter(dst io.Writer) io.WriteCloser {
	if w := pool.writers.Get(); w != nil {
		writer := w.(*snappy.Writer)
		writer.Reset(dst)
		return writer
	}
	return snappy.NewBufferedWriter(dst)
}

// PutWriter places back in the pool a CompressionWriter
func (pool *SnappyPool) PutWriter(writer io.WriteCloser) {
	pool.writers.Put(writer)
}

type NoopPool struct{}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *NoopPool) GetReader(src io.Reader) io.Reader {
	return src
}

// PutReader places back in the pool a CompressionReader
func (pool *NoopPool) PutReader(reader io.Reader) {}

type noopCloser struct {
	io.Writer
}

func (noopCloser) Close() error { return nil }

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *NoopPool) GetWriter(dst io.Writer) io.WriteCloser {
	return noopCloser{dst}
}

// PutWriter places back in the pool a CompressionWriter
func (pool *NoopPool) PutWriter(writer io.WriteCloser) {}

// BufioReaderPool is a bufio reader that uses sync.Pool.
type BufioReaderPool struct {
	pool sync.Pool
}

// Get returns a bufio.Reader which reads from r. The buffer size is that of the pool.
func (bufPool *BufioReaderPool) Get(r io.Reader) *bufio.Reader {
	buf := bufPool.pool.Get().(*bufio.Reader)
	buf.Reset(r)
	return buf
}

// Put puts the bufio.Reader back into the pool.
func (bufPool *BufioReaderPool) Put(b *bufio.Reader) {
	bufPool.pool.Put(b)
}
<commit_msg>Lint: comments<commit_after>package chunkenc

import (
	"bufio"
	"bytes"
	"io"
	"sync"

	"github.com/golang/snappy"
	"github.com/klauspost/compress/gzip"
	"github.com/pierrec/lz4"
	"github.com/prometheus/prometheus/pkg/pool"
)

// WriterPool is a pool of io.Writer
// This is used by every chunk to avoid unnecessary allocations.
type WriterPool interface {
	GetWriter(io.Writer) io.WriteCloser
	PutWriter(io.WriteCloser)
}

// ReaderPool similar to WriterPool but for reading chunks.
type ReaderPool interface {
	GetReader(io.Reader) io.Reader
	PutReader(io.Reader)
}

var (
	// Gzip is the gnu zip compression pool
	Gzip     = GzipPool{level: gzip.DefaultCompression}
	Lz4_64k  = LZ4Pool{bufferSize: 1 << 16} // Lz4_64k is the l4z compression pool, with 64k buffer size
	Lz4_256k = LZ4Pool{bufferSize: 1 << 18} // Lz4_256k uses 256k buffer
	Lz4_1M   = LZ4Pool{bufferSize: 1 << 20} // Lz4_1M uses 1M buffer
	Lz4_4M   = LZ4Pool{bufferSize: 1 << 22} // Lz4_4M uses 4M buffer

	// Snappy is the snappy compression pool
	Snappy SnappyPool
	// Noop is the no compression pool
	Noop NoopPool

	// BufReaderPool is bufio.Reader pool
	BufReaderPool = &BufioReaderPool{
		pool: sync.Pool{
			New: func() interface{} { return bufio.NewReader(nil) },
		},
	}
	// BytesBufferPool is a bytes buffer used for lines decompressed.
	// Buckets [0.5KB,1KB,2KB,4KB,8KB]
	BytesBufferPool          = pool.New(1<<9, 1<<13, 2, func(size int) interface{} { return make([]byte, 0, size) })
	serializeBytesBufferPool = sync.Pool{
		New: func() interface{} {
			return &bytes.Buffer{}
		},
	}
)

func getWriterPool(enc Encoding) WriterPool {
	return getReaderPool(enc).(WriterPool)
}

func getReaderPool(enc Encoding) ReaderPool {
	switch enc {
	case EncGZIP:
		return &Gzip
	case EncLZ4_64k:
		return &Lz4_64k
	case EncLZ4_256k:
		return &Lz4_256k
	case EncLZ4_1M:
		return &Lz4_1M
	case EncLZ4_4M:
		return &Lz4_4M
	case EncSnappy:
		return &Snappy
	case EncNone:
		return &Noop
	default:
		panic("unknown encoding")
	}
}

// GzipPool is a gun zip compression pool
type GzipPool struct {
	readers sync.Pool
	writers sync.Pool
	level   int
}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *GzipPool) GetReader(src io.Reader) io.Reader {
	if r := pool.readers.Get(); r != nil {
		reader := r.(*gzip.Reader)
		err := reader.Reset(src)
		if err != nil {
			panic(err)
		}
		return reader
	}
	reader, err := gzip.NewReader(src)
	if err != nil {
		panic(err)
	}
	return reader
}

// PutReader places back in the pool a CompressionReader
func (pool *GzipPool) PutReader(reader io.Reader) {
	pool.readers.Put(reader)
}

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *GzipPool) GetWriter(dst io.Writer) io.WriteCloser {
	if w := pool.writers.Get(); w != nil {
		writer := w.(*gzip.Writer)
		writer.Reset(dst)
		return writer
	}

	level := pool.level
	if level == 0 {
		level = gzip.DefaultCompression
	}
	w, err := gzip.NewWriterLevel(dst, level)
	if err != nil {
		panic(err) // never happens, error is only returned on wrong compression level.
	}
	return w
}

// PutWriter places back in the pool a CompressionWriter
func (pool *GzipPool) PutWriter(writer io.WriteCloser) {
	pool.writers.Put(writer)
}

type LZ4Pool struct {
	readers    sync.Pool
	writers    sync.Pool
	bufferSize int // available values: 1<<16 (64k), 1<<18 (256k), 1<<20 (1M), 1<<22 (4M). Defaults to 4MB, if not set.
}

// lz4Reader is simple wrapper around *lz4.Reader, which remembers max used block size,
// as reported by this reader. It is used to determine whether we want to reuse it,
// or throw away and garbage-collect.
type lz4Reader struct {
	r            *lz4.Reader
	maxBlockSize int
}

func (l *lz4Reader) Read(p []byte) (n int, err error) {
	return l.r.Read(p)
}

func (l *lz4Reader) Reset(src io.Reader) {
	l.r.Reset(src)
}

func (l *lz4Reader) onBlockDone(_ int) {
	// remember max block size used.
	if l.r.BlockMaxSize > l.maxBlockSize {
		l.maxBlockSize = l.r.BlockMaxSize
	}
}

func newLz4Reader(src io.Reader) *lz4Reader {
	lz4r := lz4.NewReader(src)
	r := &lz4Reader{r: lz4r}
	lz4r.OnBlockDone = r.onBlockDone
	return r
}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *LZ4Pool) GetReader(src io.Reader) io.Reader {
	if r := pool.readers.Get(); r != nil {
		reader := r.(*lz4Reader)
		reader.Reset(src)
		return reader
	}
	// no need to set buffer size here. Reader uses buffer size based on
	// LZ4 header that it is reading.
	return newLz4Reader(src)
}

// PutReader places back in the pool a CompressionReader
func (pool *LZ4Pool) PutReader(reader io.Reader) {
	r := reader.(*lz4Reader)
	if r.maxBlockSize > pool.bufferSize {
		// Readers base their buffer size based on headers from LZ4 stream.
		// If this reader uses bigger buffer than what we use currently, don't pool it.
		// Reading from a couple of chunks that used big buffer sizes could otherwise quickly lead
		// to high pooled memory usage.
		return
	}
	pool.readers.Put(reader)
}

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *LZ4Pool) GetWriter(dst io.Writer) io.WriteCloser {
	if w := pool.writers.Get(); w != nil {
		writer := w.(*lz4.Writer)
		writer.Reset(dst)
		return writer
	}
	w := lz4.NewWriter(dst)
	w.BlockMaxSize = pool.bufferSize
	return w
}

// PutWriter places back in the pool a CompressionWriter
func (pool *LZ4Pool) PutWriter(writer io.WriteCloser) {
	pool.writers.Put(writer)
}

type SnappyPool struct {
	readers sync.Pool
	writers sync.Pool
}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *SnappyPool) GetReader(src io.Reader) io.Reader {
	if r := pool.readers.Get(); r != nil {
		reader := r.(*snappy.Reader)
		reader.Reset(src)
		return reader
	}
	return snappy.NewReader(src)
}

// PutReader places back in the pool a CompressionReader
func (pool *SnappyPool) PutReader(reader io.Reader) {
	pool.readers.Put(reader)
}

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *SnappyPool) GetWriter(dst io.Writer) io.WriteCloser {
	if w := pool.writers.Get(); w != nil {
		writer := w.(*snappy.Writer)
		writer.Reset(dst)
		return writer
	}
	return snappy.NewBufferedWriter(dst)
}

// PutWriter places back in the pool a CompressionWriter
func (pool *SnappyPool) PutWriter(writer io.WriteCloser) {
	pool.writers.Put(writer)
}

type NoopPool struct{}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *NoopPool) GetReader(src io.Reader) io.Reader {
	return src
}

// PutReader places back in the pool a CompressionReader
func (pool *NoopPool) PutReader(reader io.Reader) {}

type noopCloser struct {
	io.Writer
}

func (noopCloser) Close() error { return nil }

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *NoopPool) GetWriter(dst io.Writer) io.WriteCloser {
	return noopCloser{dst}
}

// PutWriter places back in the pool a CompressionWriter
func (pool *NoopPool) PutWriter(writer io.WriteCloser) {}

// BufioReaderPool is a bufio reader that uses sync.Pool.
type BufioReaderPool struct {
	pool sync.Pool
}

// Get returns a bufio.Reader which reads from r. The buffer size is that of the pool.
func (bufPool *BufioReaderPool) Get(r io.Reader) *bufio.Reader {
	buf := bufPool.pool.Get().(*bufio.Reader)
	buf.Reset(r)
	return buf
}

// Put puts the bufio.Reader back into the pool.
func (bufPool *BufioReaderPool) Put(b *bufio.Reader) {
	bufPool.pool.Put(b)
}
<|endoftext|>


<commit_before>package config

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"text/template"

	"github.com/Masterminds/sprig"
	"gopkg.in/flant/yaml.v2"

	"github.com/flant/dapp/pkg/util"
)

func ParseDimgs(dappfilePath string) ([]*Dimg, error) {
	dappfileRenderContent, err := parseDappfileYaml(dappfilePath)
	if err != nil {
		return nil, err
	}

	dappfileRenderPath, err := dumpDappfileRender(dappfilePath, dappfileRenderContent)
	if err != nil {
		return nil, err
	}

	docs, err := splitByDocs(dappfileRenderContent, dappfileRenderPath)
	if err != nil {
		return nil, err
	}

	dimgs, err := splitByDimgs(docs, dappfileRenderContent, dappfileRenderPath)
	if err != nil {
		return nil, err
	}

	return dimgs, nil
}

func dumpDappfileRender(dappfilePath string, dappfileRenderContent string) (string, error) {
	wd, err := os.Getwd()
	if err != nil {
		return "", err
	}

	dappfileNameParts := strings.Split(path.Base(dappfilePath), ".")
	dappfileRenderNameParts := []string{}
	dappfileRenderNameParts = append(dappfileRenderNameParts, dappfileNameParts[0:len(dappfileNameParts)-1]...)
	dappfileRenderNameParts = append(dappfileRenderNameParts, "render", dappfileNameParts[len(dappfileNameParts)-1])
	dappfileRenderPath := path.Join(wd, fmt.Sprintf(".%s", strings.Join(dappfileRenderNameParts, ".")))

	dappfileRenderFile, err := os.OpenFile(dappfileRenderPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return "", err
	}
	dappfileRenderFile.Write([]byte(dappfileRenderContent))
	dappfileRenderFile.Close()

	return dappfileRenderPath, nil
}

func splitByDocs(dappfileRenderContent string, dappfileRenderPath string) ([]*Doc, error) {
	scanner := bufio.NewScanner(strings.NewReader(dappfileRenderContent))
	scanner.Split(splitYAMLDocument)

	var docs []*Doc
	var line int
	for scanner.Scan() {
		content := scanner.Bytes()

		if strings.TrimSpace(string(content)) != "" {
			docs = append(docs, &Doc{
				Line:           line,
				Content:        content,
				RenderFilePath: dappfileRenderPath,
			})
		}

		contentLines := bytes.Split(content, []byte("\n"))
		if string(contentLines[len(contentLines)-1]) == "" {
			contentLines = contentLines[0 : len(contentLines)-1]
		}
		line += len(contentLines) + 1
	}

	return docs, nil
}

// TODO: переделать на ParseFiles вместо Parse
func parseDappfileYaml(dappfileRenderContent string) (string, error) {
	data, err := ioutil.ReadFile(dappfileRenderContent)
	if err != nil {
		return "", err
	}

	tmpl := template.New("dappfile")
	tmpl.Funcs(funcMap(tmpl))
	if _, err := tmpl.Parse(string(data)); err != nil {
		return "", err
	}
	return executeTemplate(tmpl, "dappfile", nil)
}

func funcMap(tmpl *template.Template) template.FuncMap {
	funcMap := sprig.TxtFuncMap()
	funcMap["include"] = func(name string, data interface{}) (string, error) {
		return executeTemplate(tmpl, name, data)
	}
	return funcMap
}

func executeTemplate(tmpl *template.Template, name string, data interface{}) (string, error) {
	buf := bytes.NewBuffer(nil)
	if err := tmpl.ExecuteTemplate(buf, name, data); err != nil {
		return "", err
	}
	return buf.String(), nil
}

func splitYAMLDocument(data []byte, atEOF bool) (advance int, token []byte, err error) {
	yamlSeparator := "---"

	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	sep := len([]byte(yamlSeparator))
	if i := bytes.Index(data, []byte(yamlSeparator)); i >= 0 {
		i += sep
		after := data[i:]
		if len(after) == 0 {
			if atEOF {
				return len(data), data[:len(data)-sep], nil
			}
			return 0, nil, nil
		}
		if j := bytes.IndexByte(after, '\n'); j >= 0 {
			return i + j + 1, data[0 : i-sep], nil
		}
		return 0, nil, nil
	}
	if atEOF {
		return len(data), data, nil
	}
	return 0, nil, nil
}

func splitByDimgs(docs []*Doc, dappfileRenderContent string, dappfileRenderPath string) ([]*Dimg, error) {
	rawDimgs, err := splitByRawDimgs(docs)
	if err != nil {
		return nil, err
	}

	var dimgs []*Dimg
	var artifacts []*DimgArtifact

	for _, rawDimg := range rawDimgs {
		if rawDimg.Type() == "dimg" {
			if dimg, err := rawDimg.ToDirective(); err != nil {
				return nil, err
			} else {
				dimgs = append(dimgs, dimg)
			}
		} else {
			if dimgArtifact, err := rawDimg.ToArtifactDirective(); err != nil {
				return nil, err
			} else {
				artifacts = append(artifacts, dimgArtifact)
			}
		}
	}

	if len(dimgs) == 0 {
		return nil, fmt.Errorf("No dimgs defined, at least one dimg required!\n\n%s:\n\n```\n%s```\n", dappfileRenderPath, dappfileRenderContent)
	}

	if err = associateArtifacts(dimgs, artifacts); err != nil {
		return nil, err
	}

	return dimgs, nil
}

func associateArtifacts(dimgs []*Dimg, artifacts []*DimgArtifact) error {
	for _, dimg := range dimgs {
		for _, importArtifact := range dimg.Import {
			if err := importArtifact.AssociateArtifact(artifacts); err != nil {
				return err
			}
		}
	}
	for _, dimg := range artifacts {
		for _, importArtifact := range dimg.Import {
			if err := importArtifact.AssociateArtifact(artifacts); err != nil {
				return err
			}
		}
	}
	return nil
}

func splitByRawDimgs(docs []*Doc) ([]*RawDimg, error) {
	var rawDimgs []*RawDimg
	ParentStack = util.NewStack()
	for _, doc := range docs {
		dimg := &RawDimg{Doc: doc}
		err := yaml.Unmarshal(doc.Content, &dimg)
		if err != nil {
			return nil, err
		}
		rawDimgs = append(rawDimgs, dimg)
	}

	return rawDimgs, nil
}
<commit_msg>Config yaml: fix разделение yaml на doc-и<commit_after>package config

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"text/template"

	"github.com/Masterminds/sprig"
	"gopkg.in/flant/yaml.v2"

	"github.com/flant/dapp/pkg/util"
)

func ParseDimgs(dappfilePath string) ([]*Dimg, error) {
	dappfileRenderContent, err := parseDappfileYaml(dappfilePath)
	if err != nil {
		return nil, err
	}

	dappfileRenderPath, err := dumpDappfileRender(dappfilePath, dappfileRenderContent)
	if err != nil {
		return nil, err
	}

	docs, err := splitByDocs(dappfileRenderContent, dappfileRenderPath)
	if err != nil {
		return nil, err
	}

	dimgs, err := splitByDimgs(docs, dappfileRenderContent, dappfileRenderPath)
	if err != nil {
		return nil, err
	}

	return dimgs, nil
}

func dumpDappfileRender(dappfilePath string, dappfileRenderContent string) (string, error) {
	wd, err := os.Getwd()
	if err != nil {
		return "", err
	}

	dappfileNameParts := strings.Split(path.Base(dappfilePath), ".")
	dappfileRenderNameParts := []string{}
	dappfileRenderNameParts = append(dappfileRenderNameParts, dappfileNameParts[0:len(dappfileNameParts)-1]...)
	dappfileRenderNameParts = append(dappfileRenderNameParts, "render", dappfileNameParts[len(dappfileNameParts)-1])
	dappfileRenderPath := path.Join(wd, fmt.Sprintf(".%s", strings.Join(dappfileRenderNameParts, ".")))

	dappfileRenderFile, err := os.OpenFile(dappfileRenderPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return "", err
	}
	dappfileRenderFile.Write([]byte(dappfileRenderContent))
	dappfileRenderFile.Close()

	return dappfileRenderPath, nil
}

func splitByDocs(dappfileRenderContent string, dappfileRenderPath string) ([]*Doc, error) {
	scanner := bufio.NewScanner(strings.NewReader(dappfileRenderContent))
	scanner.Split(splitYAMLDocument)

	var docs []*Doc
	var line int
	for scanner.Scan() {
		content := make([]byte, len(scanner.Bytes()))
		copy(content, scanner.Bytes())

		if strings.TrimSpace(string(content)) != "" {
			docs = append(docs, &Doc{
				Line:           line,
				Content:        content,
				RenderFilePath: dappfileRenderPath,
			})
		}

		contentLines := bytes.Split(content, []byte("\n"))
		if string(contentLines[len(contentLines)-1]) == "" {
			contentLines = contentLines[0 : len(contentLines)-1]
		}
		line += len(contentLines) + 1
	}

	return docs, nil
}

// TODO: переделать на ParseFiles вместо Parse
func parseDappfileYaml(dappfileRenderContent string) (string, error) {
	data, err := ioutil.ReadFile(dappfileRenderContent)
	if err != nil {
		return "", err
	}

	tmpl := template.New("dappfile")
	tmpl.Funcs(funcMap(tmpl))
	if _, err := tmpl.Parse(string(data)); err != nil {
		return "", err
	}
	return executeTemplate(tmpl, "dappfile", nil)
}

func funcMap(tmpl *template.Template) template.FuncMap {
	funcMap := sprig.TxtFuncMap()
	funcMap["include"] = func(name string, data interface{}) (string, error) {
		return executeTemplate(tmpl, name, data)
	}
	return funcMap
}

func executeTemplate(tmpl *template.Template, name string, data interface{}) (string, error) {
	buf := bytes.NewBuffer(nil)
	if err := tmpl.ExecuteTemplate(buf, name, data); err != nil {
		return "", err
	}
	return buf.String(), nil
}

func splitYAMLDocument(data []byte, atEOF bool) (advance int, token []byte, err error) {
	yamlSeparator := "---"

	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	sep := len([]byte(yamlSeparator))
	if i := bytes.Index(data, []byte(yamlSeparator)); i >= 0 {
		i += sep
		after := data[i:]
		if len(after) == 0 {
			if atEOF {
				return len(data), data[:len(data)-sep], nil
			}
			return 0, nil, nil
		}
		if j := bytes.IndexByte(after, '\n'); j >= 0 {
			return i + j + 1, data[0 : i-sep], nil
		}
		return 0, nil, nil
	}
	if atEOF {
		return len(data), data, nil
	}
	return 0, nil, nil
}

func splitByDimgs(docs []*Doc, dappfileRenderContent string, dappfileRenderPath string) ([]*Dimg, error) {
	rawDimgs, err := splitByRawDimgs(docs)
	if err != nil {
		return nil, err
	}

	var dimgs []*Dimg
	var artifacts []*DimgArtifact

	for _, rawDimg := range rawDimgs {
		if rawDimg.Type() == "dimg" {
			if dimg, err := rawDimg.ToDirective(); err != nil {
				return nil, err
			} else {
				dimgs = append(dimgs, dimg)
			}
		} else {
			if dimgArtifact, err := rawDimg.ToArtifactDirective(); err != nil {
				return nil, err
			} else {
				artifacts = append(artifacts, dimgArtifact)
			}
		}
	}

	if len(dimgs) == 0 {
		return nil, fmt.Errorf("No dimgs defined, at least one dimg required!\n\n%s:\n\n```\n%s```\n", dappfileRenderPath, dappfileRenderContent)
	}

	if err = associateArtifacts(dimgs, artifacts); err != nil {
		return nil, err
	}

	return dimgs, nil
}

func associateArtifacts(dimgs []*Dimg, artifacts []*DimgArtifact) error {
	for _, dimg := range dimgs {
		for _, importArtifact := range dimg.Import {
			if err := importArtifact.AssociateArtifact(artifacts); err != nil {
				return err
			}
		}
	}
	for _, dimg := range artifacts {
		for _, importArtifact := range dimg.Import {
			if err := importArtifact.AssociateArtifact(artifacts); err != nil {
				return err
			}
		}
	}
	return nil
}

func splitByRawDimgs(docs []*Doc) ([]*RawDimg, error) {
	var rawDimgs []*RawDimg
	ParentStack = util.NewStack()
	for _, doc := range docs {
		dimg := &RawDimg{Doc: doc}
		err := yaml.Unmarshal(doc.Content, &dimg)
		if err != nil {
			return nil, err
		}
		rawDimgs = append(rawDimgs, dimg)
	}

	return rawDimgs, nil
}
<|endoftext|>