<commit_before>package shellexec

import (
	"bytes"
	"errors"
	"os"
	"os/exec"
	"unicode"
	"unicode/utf8"
)

var (
	ErrEmptyCommand       = errors.New("empty command")
	ErrUnterminatedString = errors.New("string not terminated")
)

// Command parses line using a shell-like syntax and returns
// the os/exec.Cmd struct to execute the line.
func Command(line string) (*exec.Cmd, error) {
	s := parser{s: line, getenv: os.Getenv}

	c, err := s.parseLine()
	if err != nil {
		return nil, err
	}
	cmd := exec.Command(c.cmd, c.args...)
	cmd.Env = c.env
	return cmd, nil
}

type cmd struct {
	cmd  string
	args []string
	env  []string
}

type parser struct {
	buf    bytes.Buffer
	s      string
	last   rune
	peeked *rune
	getenv func(key string) string

	identBuf bytes.Buffer
}

const eof rune = utf8.MaxRune + 1

func (p *parser) next() rune {
	if p.peeked != nil {
		r := *p.peeked
		p.peeked = nil
		return r
	}

	if len(p.s) == 0 {
		p.last = eof
		return eof
	}
	var size int
	p.last, size = utf8.DecodeRuneInString(p.s)
	p.s = p.s[size:]
	return p.last
}

func (p *parser) backup() {
	p.peeked = &p.last
}

func (p *parser) token() string {
	t := p.buf.String()
	p.buf.Reset()
	return t
}

func (p *parser) parseLine() (cmd, error) {
	var c cmd
loop:
	for {
		r := p.next()
		switch {
		case unicode.IsSpace(r):
			continue
		case r == eof:
			break loop
		}
		p.backup()

		if c.cmd == "" {
			if err := p.parseVarAssign(); err == errNotAssign {
				c.cmd = p.token()
			} else if err != nil {
				return cmd{}, err
			} else {
				c.env = append(c.env, p.token())
			}
		} else {
			if err := p.parseField(); err != nil {
				return cmd{}, err
			}
			c.args = append(c.args, p.token())
		}
	}
	if c.cmd == "" {
		return cmd{}, ErrEmptyCommand
	}
	return c, nil
}

var errNotAssign = errors.New("not assignment")

func (p *parser) parseVarAssign() error {
	v := p.parseIdent()
	p.buf.WriteString(v)
	err := errNotAssign
	if v != "" && p.next() == '=' {
		err = nil
	}
	p.backup()

	if err := p.parseField(); err != nil {
		return err
	}
	return err
}

func (p *parser) parseField() error {
	esc := false
	for {
		r := p.next()
		if r == eof {
			break
		}

		if esc {
			p.buf.WriteRune(r)
			esc = false
			continue
		}
		if unicode.IsSpace(r) {
			break
		}
		switch r {
		case '\'':
			if err := p.parseSingleQuotes(); err != nil {
				return err
			}
		case '"':
			if err := p.parseDoubleQuotes(); err != nil {
				return err
			}
		case '\\':
			esc = true
			continue
		case '$':
			if err := p.parseVarExpr(); err != nil {
				return err
			}
			p.backup()
			continue
		case '|', '&', ';', '<', '>', '(', ')', '`',
			// Forbid these characters as they may need to be
			// quoted under certain circumstances.
			'*', '?', '[', '#', '~':
			return errors.New("unsupported character: " + string(r))
		default:
			p.buf.WriteRune(r)
		}
	}
	return nil
}

func (p *parser) parseSingleQuotes() error {
	for {
		switch r := p.next(); r {
		case '\'':
			return nil
		case eof:
			return ErrUnterminatedString
		default:
			p.buf.WriteRune(r)
		}
	}
	panic("unreachable")
}

func (p *parser) parseDoubleQuotes() error {
	var esc bool
	for {
		r := p.next()
		if r == eof {
			return ErrUnterminatedString
		}

		if esc {
			switch r {
			default:
				p.buf.WriteRune('\\')
				fallthrough
			case '$', '`', '"', '\\':
				p.buf.WriteRune(r)
			case '\n':
				// Do nothing.
			}
			esc = false
			continue
		}
		switch r {
		case '"':
			return nil
		case '\\':
			esc = true
			continue
		case '$':
			if err := p.parseVarExpr(); err != nil {
				return err
			}
			continue
		case '`':
			return errors.New("unsupported character inside string: " + string(r))
		}
		p.buf.WriteRune(r)
	}
	panic("unreachable")
}

func (p *parser) parseVarExpr() error {
	switch p.next() {
	case '(':
		return errors.New("command substitution or arithmetic expansion not supported")
	case '{':
		return errors.New("parameter expansion not supported")
	case '@', '*', '#', '?', '-', '$', '!', '0':
		return errors.New("special parameters not supported")
	case '1', '2', '3', '4', '5', '6', '7', '8', '9':
		return errors.New("positional parameters not supported")
	}
	p.backup()

	v := "$"
	if name := p.parseIdent(); name != "" {
		v = p.getenv(name)
	}
	p.buf.WriteString(v)
	return nil
}

func (p *parser) parseIdent() string {
	p.identBuf.Reset()
	for {
		r := p.next()
		if !(r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z' || r == '_' ||
			p.identBuf.Len() > 0 && r >= '0' && r <= '9') {
			p.backup()
			return p.identBuf.String()
		}
		p.identBuf.WriteRune(r)
	}
	panic("unreachable")
}
<commit_msg>Get rid of global error variables<commit_after>package shellexec

import (
	"bytes"
	"errors"
	"os"
	"os/exec"
	"unicode"
	"unicode/utf8"
)

// Command parses line using a shell-like syntax and returns
// the os/exec.Cmd struct to execute the line.
func Command(line string) (*exec.Cmd, error) {
	s := parser{s: line, getenv: os.Getenv}

	c, err := s.parseLine()
	if err != nil {
		return nil, err
	}
	cmd := exec.Command(c.cmd, c.args...)
	cmd.Env = c.env
	return cmd, nil
}

type cmd struct {
	cmd  string
	args []string
	env  []string
}

type parser struct {
	buf    bytes.Buffer
	s      string
	last   rune
	peeked *rune
	getenv func(key string) string

	identBuf bytes.Buffer
}

const eof rune = utf8.MaxRune + 1

func (p *parser) next() rune {
	if p.peeked != nil {
		r := *p.peeked
		p.peeked = nil
		return r
	}

	if len(p.s) == 0 {
		p.last = eof
		return eof
	}
	var size int
	p.last, size = utf8.DecodeRuneInString(p.s)
	p.s = p.s[size:]
	return p.last
}

func (p *parser) backup() {
	p.peeked = &p.last
}

func (p *parser) token() string {
	t := p.buf.String()
	p.buf.Reset()
	return t
}

func (p *parser) parseLine() (cmd, error) {
	var c cmd
loop:
	for {
		r := p.next()
		switch {
		case unicode.IsSpace(r):
			continue
		case r == eof:
			break loop
		}
		p.backup()

		if c.cmd == "" {
			if err := p.parseVarAssign(); err == errNotAssign {
				c.cmd = p.token()
			} else if err != nil {
				return cmd{}, err
			} else {
				c.env = append(c.env, p.token())
			}
		} else {
			if err := p.parseField(); err != nil {
				return cmd{}, err
			}
			c.args = append(c.args, p.token())
		}
	}
	if c.cmd == "" {
		return cmd{}, errors.New("empty command")
	}
	return c, nil
}

var errNotAssign = errors.New("not assignment")

func (p *parser) parseVarAssign() error {
	v := p.parseIdent()
	p.buf.WriteString(v)
	err := errNotAssign
	if v != "" && p.next() == '=' {
		err = nil
	}
	p.backup()

	if err := p.parseField(); err != nil {
		return err
	}
	return err
}

func (p *parser) parseField() error {
	esc := false
	for {
		r := p.next()
		if r == eof {
			break
		}

		if esc {
			p.buf.WriteRune(r)
			esc = false
			continue
		}
		if unicode.IsSpace(r) {
			break
		}
		switch r {
		case '\'':
			if err := p.parseSingleQuotes(); err != nil {
				return err
			}
		case '"':
			if err := p.parseDoubleQuotes(); err != nil {
				return err
			}
		case '\\':
			esc = true
			continue
		case '$':
			if err := p.parseVarExpr(); err != nil {
				return err
			}
			p.backup()
			continue
		case '|', '&', ';', '<', '>', '(', ')', '`',
			// Forbid these characters as they may need to be
			// quoted under certain circumstances.
			'*', '?', '[', '#', '~':
			return errors.New("unsupported character: " + string(r))
		default:
			p.buf.WriteRune(r)
		}
	}
	return nil
}

func (p *parser) parseSingleQuotes() error {
	for {
		switch r := p.next(); r {
		case '\'':
			return nil
		case eof:
			return errors.New("string not terminated")
		default:
			p.buf.WriteRune(r)
		}
	}
	panic("unreachable")
}

func (p *parser) parseDoubleQuotes() error {
	var esc bool
	for {
		r := p.next()
		if r == eof {
			return errors.New("string not terminated")
		}

		if esc {
			switch r {
			default:
				p.buf.WriteRune('\\')
				fallthrough
			case '$', '`', '"', '\\':
				p.buf.WriteRune(r)
			case '\n':
				// Do nothing.
			}
			esc = false
			continue
		}
		switch r {
		case '"':
			return nil
		case '\\':
			esc = true
			continue
		case '$':
			if err := p.parseVarExpr(); err != nil {
				return err
			}
			continue
		case '`':
			return errors.New("unsupported character inside string: " + string(r))
		}
		p.buf.WriteRune(r)
	}
	panic("unreachable")
}

func (p *parser) parseVarExpr() error {
	switch p.next() {
	case '(':
		return errors.New("command substitution or arithmetic expansion not supported")
	case '{':
		return errors.New("parameter expansion not supported")
	case '@', '*', '#', '?', '-', '$', '!', '0':
		return errors.New("special parameters not supported")
	case '1', '2', '3', '4', '5', '6', '7', '8', '9':
		return errors.New("positional parameters not supported")
	}
	p.backup()

	v := "$"
	if name := p.parseIdent(); name != "" {
		v = p.getenv(name)
	}
	p.buf.WriteString(v)
	return nil
}

func (p *parser) parseIdent() string {
	p.identBuf.Reset()
	for {
		r := p.next()
		if !(r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z' || r == '_' ||
			p.identBuf.Len() > 0 && r >= '0' && r <= '9') {
			p.backup()
			return p.identBuf.String()
		}
		p.identBuf.WriteRune(r)
	}
	panic("unreachable")
}
<|endoftext|>


<commit_before>// Copyright 2015 The rpcmq Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpcmq

import (
	"crypto/tls"
	"fmt"
	"sync"
	"time"

	"github.com/streadway/amqp"
)

// RetrySleep is the time between retries if the connection with the broker
// is lost. The default value is 2 seconds.
var RetrySleep time.Duration = 2 * time.Second

type publishing struct {
	msg       amqp.Publishing
	timestamp time.Time
}

type amqpClient struct {
	uri         string
	consumerTag string

	conn    *amqp.Connection
	channel *amqp.Channel
	done    chan bool

	// The mutex protects the acks/nacks channels from being used
	// simultaneously by multiple publishing processes
	mu          sync.Mutex
	acks, nacks chan uint64

	setupFunc func() error

	tlsConfig *tls.Config
}

func newAmqpClient(uri string) *amqpClient {
	ac := &amqpClient{
		uri:  uri,
		done: make(chan bool),
	}
	return ac
}

func (ac *amqpClient) init() error {
	var err error
	ac.conn, err = amqp.DialTLS(ac.uri, ac.tlsConfig)
	if err != nil {
		return fmt.Errorf("DialTLS: %v", err)
	}

	ac.channel, err = ac.conn.Channel()
	if err != nil {
		return fmt.Errorf("Channel: %v", err)
	}
	if err := ac.channel.Confirm(false); err != nil {
		return fmt.Errorf("Confirm: %v", err)
	}
	ac.acks, ac.nacks = ac.channel.NotifyConfirm(make(chan uint64), make(chan uint64))

	go ac.handleMsgs()

	return nil
}

func (ac *amqpClient) handleMsgs() {
	returns := ac.channel.NotifyReturn(make(chan amqp.Return))
	errors := ac.channel.NotifyClose(make(chan *amqp.Error))

	for {
		select {
		case ret, ok := <-returns:
			if !ok {
				break
			}
			if ret.ReplyCode == amqp.NoRoute {
				panic("no route")
			}
		case _, ok := <-errors:
			if !ok {
				break
			}
			logf("shutdown")
			ac.shutdown()
			for {
				logf("retrying")
				if err := ac.init(); err != nil {
					logf("amqpClient init: %v", err)
					time.Sleep(RetrySleep)
					continue
				}

				if err := ac.setupFunc(); err != nil {
					panic(fmt.Errorf("setup: %v", err))
				}
				logf("connected")
				return
			}
		}
	}
}

func (ac *amqpClient) shutdown() {
	if ac.consumerTag != "" {
		if err := ac.channel.Cancel(ac.consumerTag, false); err != nil {
			logf("Channel Cancel: %v", err)
		}
	}
	<-ac.done
	if err := ac.channel.Close(); err != nil {
		logf("Channel Close: %v", err)
	}
	if err := ac.conn.Close(); err != nil {
		logf("Connection Close: %v", err)
	}
}
<commit_msg>Update docs<commit_after>// Copyright 2015 The rpcmq Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpcmq

import (
	"crypto/tls"
	"fmt"
	"sync"
	"time"

	"github.com/streadway/amqp"
)

// RetrySleep is the time between retries if the connection with the broker
// is lost.
var RetrySleep time.Duration = 2 * time.Second

type publishing struct {
	msg       amqp.Publishing
	timestamp time.Time
}

type amqpClient struct {
	uri         string
	consumerTag string

	conn    *amqp.Connection
	channel *amqp.Channel
	done    chan bool

	// The mutex protects the acks/nacks channels from being used
	// simultaneously by multiple publishing processes
	mu          sync.Mutex
	acks, nacks chan uint64

	setupFunc func() error

	tlsConfig *tls.Config
}

func newAmqpClient(uri string) *amqpClient {
	ac := &amqpClient{
		uri:  uri,
		done: make(chan bool),
	}
	return ac
}

func (ac *amqpClient) init() error {
	var err error
	ac.conn, err = amqp.DialTLS(ac.uri, ac.tlsConfig)
	if err != nil {
		return fmt.Errorf("DialTLS: %v", err)
	}

	ac.channel, err = ac.conn.Channel()
	if err != nil {
		return fmt.Errorf("Channel: %v", err)
	}
	if err := ac.channel.Confirm(false); err != nil {
		return fmt.Errorf("Confirm: %v", err)
	}
	ac.acks, ac.nacks = ac.channel.NotifyConfirm(make(chan uint64), make(chan uint64))

	go ac.handleMsgs()

	return nil
}

func (ac *amqpClient) handleMsgs() {
	returns := ac.channel.NotifyReturn(make(chan amqp.Return))
	errors := ac.channel.NotifyClose(make(chan *amqp.Error))

	for {
		select {
		case ret, ok := <-returns:
			if !ok {
				break
			}
			if ret.ReplyCode == amqp.NoRoute {
				panic("no route")
			}
		case _, ok := <-errors:
			if !ok {
				break
			}
			logf("shutdown")
			ac.shutdown()
			for {
				logf("retrying")
				if err := ac.init(); err != nil {
					logf("amqpClient init: %v", err)
					time.Sleep(RetrySleep)
					continue
				}

				if err := ac.setupFunc(); err != nil {
					panic(fmt.Errorf("setup: %v", err))
				}
				logf("connected")
				return
			}
		}
	}
}

func (ac *amqpClient) shutdown() {
	if ac.consumerTag != "" {
		if err := ac.channel.Cancel(ac.consumerTag, false); err != nil {
			logf("Channel Cancel: %v", err)
		}
	}
	<-ac.done
	if err := ac.channel.Close(); err != nil {
		logf("Channel Close: %v", err)
	}
	if err := ac.conn.Close(); err != nil {
		logf("Connection Close: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"time"

	docker_types "github.com/docker/docker/api/types"
	docker_types_container "github.com/docker/docker/api/types/container"
	docker_types_network "github.com/docker/docker/api/types/network"
)

// DockerPorts describes docker ports for docker run
type DockerPorts struct {
	HostPort      int    `json:"host_port"`
	ContainerPort int    `json:"container_port"`
	Protocol      string `json:"protocol"`
}

// DockerStartOpts describes docker start options
type DockerStartOpts struct {
}

// DockerCreateOpts describes docker run options
type DockerCreateOpts struct {
	Name       string         `json:"name"`
	Image      string         `json:"image"`
	Remove     bool           `json:"rm"`
	Ports      []*DockerPorts `json:"ports"`
	PublishAll bool           `json:"publish_all"`
	Command    string         `json:"command"`
}

// DockerPullOpts describes docker pull options
type DockerPullOpts struct {
	Image string `json:"image"`
}

// AgentDockerInfo describes docker info
type AgentDockerInfo struct {
	Info docker_types.Info `json:"info"`
}

// AgentContainer describes docker container
type AgentContainer struct {
	ID   string `json:"cid"`
	Data string `json:"data"`
}

func (agent *Agent) dockerPull(opts *DockerPullOpts) error {
	ctx := context.Background()
	pullOpts := docker_types.ImagePullOptions{}
	out, err := agent.DockerClient.ImagePull(ctx, opts.Image, pullOpts)
	if err != nil {
		return err
	}
	defer out.Close()
	log.Printf("New image pulled %s", opts.Image)
	return nil
}

func (agent *Agent) dockerCreate(opts *DockerCreateOpts) (*docker_types_container.ContainerCreateCreatedBody, error) {
	ctx := context.Background()
	config := &docker_types_container.Config{
		Image: opts.Image,
	}
	hostConfig := &docker_types_container.HostConfig{}
	networkingConfig := &docker_types_network.NetworkingConfig{}

	container, err := agent.DockerClient.ContainerCreate(ctx, config, hostConfig, networkingConfig, opts.Name)
	if err != nil {
		return nil, err
	}
	log.Printf("New container created %s", container.ID)
	return &container, nil
}

func (agent *Agent) dockerStart(containerID string) error {
	ctx := context.Background()
	startOpts := docker_types.ContainerStartOptions{}
	if err := agent.DockerClient.ContainerStart(ctx, containerID, startOpts); err != nil {
		return err
	}
	log.Printf("New container started %s", containerID)
	return nil
}

func (agent *Agent) syncDockerInfo() {
	for {
		uri := fmt.Sprintf("run/agents/%s/docker/info/", agent.ID)
		info, _ := agent.DockerClient.Info(context.Background())
		updateInfo := AgentDockerInfo{
			Info: info,
		}
		status, err := agent.Client.Put(uri, updateInfo, nil)
		if err != nil {
			log.Println(err.Error())
		}
		if status != 200 {
			log.Printf("Failed to push docker info: %d", status)
		} else {
			log.Println("Sync docker info OK")
		}
		time.Sleep(3 * time.Second)
	}
}

func (agent *Agent) syncDockerContainers() {
	containersListOpts := docker_types.ContainerListOptions{
		All: true,
	}
	for {
		uri := fmt.Sprintf("run/agents/%s/docker/containers/", agent.ID)
		containers, _ := agent.DockerClient.ContainerList(context.Background(), containersListOpts)
		var containersCreateList []AgentContainer
		for _, container := range containers {
			data, err := json.Marshal(container)
			if err != nil {
				log.Printf("Cannot decode %v", container)
				continue
			}
			containersCreateList = append(containersCreateList, AgentContainer{ID: container.ID, Data: string(data)})
		}
		status, err := agent.Client.Post(uri, containersCreateList, nil)
		if err != nil {
			log.Println(err.Error())
		}
		if status != 200 {
			log.Printf("Failed to push docker containers: %d", status)
		} else {
			log.Printf("Sync docker %d containers OK", len(containersCreateList))
		}
		time.Sleep(3 * time.Second)
	}
}

// Run docker container
func (agent *Agent) Run() {

}
<commit_msg>push container config<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"time"

	docker_types "github.com/docker/docker/api/types"
	docker_types_container "github.com/docker/docker/api/types/container"
	docker_types_network "github.com/docker/docker/api/types/network"
)

// DockerPorts describes docker ports for docker run
type DockerPorts struct {
	HostPort      int    `json:"host_port"`
	ContainerPort int    `json:"container_port"`
	Protocol      string `json:"protocol"`
}

// DockerStartOpts describes docker start options
type DockerStartOpts struct {
}

// DockerCreateOpts describes docker run options
type DockerCreateOpts struct {
	Name       string         `json:"name"`
	Image      string         `json:"image"`
	Remove     bool           `json:"rm"`
	Ports      []*DockerPorts `json:"ports"`
	PublishAll bool           `json:"publish_all"`
	Command    string         `json:"command"`
}

// DockerPullOpts describes docker pull options
type DockerPullOpts struct {
	Image string `json:"image"`
}

// AgentDockerInfo describes docker info
type AgentDockerInfo struct {
	Info docker_types.Info `json:"info"`
}

// AgentContainer describes docker container
type AgentContainer struct {
	ID        string `json:"cid"`
	Container string `json:"container"`
	Config    string `json:"config"`
}

func (agent *Agent) dockerPull(opts *DockerPullOpts) error {
	ctx := context.Background()
	pullOpts := docker_types.ImagePullOptions{}
	out, err := agent.DockerClient.ImagePull(ctx, opts.Image, pullOpts)
	if err != nil {
		return err
	}
	defer out.Close()
	log.Printf("New image pulled %s", opts.Image)
	return nil
}

func (agent *Agent) dockerCreate(opts *DockerCreateOpts) (*docker_types_container.ContainerCreateCreatedBody, error) {
	ctx := context.Background()
	config := &docker_types_container.Config{
		Image: opts.Image,
	}
	hostConfig := &docker_types_container.HostConfig{}
	networkingConfig := &docker_types_network.NetworkingConfig{}

	container, err := agent.DockerClient.ContainerCreate(ctx, config, hostConfig, networkingConfig, opts.Name)
	if err != nil {
		return nil, err
	}
	log.Printf("New container created %s", container.ID)
	return &container, nil
}

func (agent *Agent) dockerStart(containerID string) error {
	ctx := context.Background()
	startOpts := docker_types.ContainerStartOptions{}
	if err := agent.DockerClient.ContainerStart(ctx, containerID, startOpts); err != nil {
		return err
	}
	log.Printf("New container started %s", containerID)
	return nil
}

func (agent *Agent) syncDockerInfo() {
	for {
		uri := fmt.Sprintf("run/agents/%s/docker/info/", agent.ID)
		info, _ := agent.DockerClient.Info(context.Background())
		updateInfo := AgentDockerInfo{
			Info: info,
		}
		status, err := agent.Client.Put(uri, updateInfo, nil)
		if err != nil {
			log.Println(err.Error())
		}
		if status != 200 {
			log.Printf("Failed to push docker info: %d", status)
		} else {
			log.Println("Sync docker info OK")
		}
		time.Sleep(3 * time.Second)
	}
}

func (agent *Agent) syncDockerContainers() {
	containersListOpts := docker_types.ContainerListOptions{
		All: true,
	}
	for {
		uri := fmt.Sprintf("run/agents/%s/docker/containers/", agent.ID)
		containers, _ := agent.DockerClient.ContainerList(context.Background(), containersListOpts)
		var containersCreateList []AgentContainer
		for _, container := range containers {
			data, err := json.Marshal(container)
			if err != nil {
				log.Printf("Cannot decode %v", container)
				continue
			}
			inspect, err := agent.DockerClient.ContainerInspect(context.Background(), container.ID)
			if err != nil {
				log.Printf("Cannot inspect container %v", err)
			}
			inspectConfig, err := json.Marshal(inspect)
			if err != nil {
				log.Printf("Cannot decode %v", inspect)
				continue
			}
			containersCreateList = append(containersCreateList,
				AgentContainer{
					ID:        container.ID,
					Container: string(data),
					Config:    string(inspectConfig),
				})
		}
		status, err := agent.Client.Post(uri, containersCreateList, nil)
		if err != nil {
			log.Println(err.Error())
		}
		if status != 200 {
			log.Printf("Failed to push docker containers: %d", status)
		} else {
			log.Printf("Sync docker %d containers OK", len(containersCreateList))
		}
		time.Sleep(3 * time.Second)
	}
}

// Run docker container
func (agent *Agent) Run() {

}
<|endoftext|>


<commit_before>package verifier

import (
	"crypto/md5"
	"encoding/hex"
	"net/mail"
	"strings"
)

// Address stores all information about an email Address
type Address struct{ Username, Domain, Address string }

// ParseAddress attempts to parse an email address and return it in the form
// of an Address struct pointer
func ParseAddress(email string) (*Address, error) {
	// Parses the address with the internal go mail address parser
	a, err := mail.ParseAddress(strings.ToLower(email))
	if err != nil {
		return nil, err
	}

	// Returns the address with the username and domain split out
	return &Address{
		Username: strings.Split(a.Address, "@")[0],
		Domain:   strings.Split(a.Address, "@")[1],
		Address:  a.Address,
	}, nil
}

// MD5 takes a calculation of the md5 Hash and returns a string representation
func (a *Address) MD5() string {
	hashBytes := md5.Sum([]byte(a.Address))
	return hex.EncodeToString(hashBytes[:])
}
<commit_msg>Splitting on the final @ sign<commit_after>package verifier

import (
	"crypto/md5"
	"encoding/hex"
	"net/mail"
	"strings"
)

// Address stores all information about an email Address
type Address struct{ Username, Domain, Address string }

// ParseAddress attempts to parse an email address and return it in the form
// of an Address struct pointer
func ParseAddress(email string) (*Address, error) {
	// Parses the address with the internal go mail address parser
	a, err := mail.ParseAddress(strings.ToLower(email))
	if err != nil {
		return nil, err
	}

	// Find the last occurrence of an @ sign
	index := strings.LastIndex(a.Address, "@")

	// Returns the address with the username and domain split out
	return &Address{
		Username: a.Address[:index],
		Domain:   a.Address[index+1:],
		Address:  a.Address,
	}, nil
}

// MD5 takes a calculation of the md5 Hash and returns a string representation
func (a *Address) MD5() string {
	hashBytes := md5.Sum([]byte(a.Address))
	return hex.EncodeToString(hashBytes[:])
}
<|endoftext|>


<commit_before>package sakuracloud

var Version = "0.0.2"
<commit_msg>Bump to v0.0.3<commit_after>package sakuracloud

var Version = "0.0.3"
<|endoftext|>


<commit_before>package main

import (
	"log"
)

type AtomData struct {

}

type AtomFeed struct {

}

type AtomItemData struct {

}

func readAtom(feed *Feed) {
	url := feed.Info.URL
	log.Print(url)
	r, err := client.Get(url)
	if err != nil {
		log.Print("ERROR: ", err.Error())
	}
	_ = r.Body

	changed := false

	if (changed) {
		updates <- DoUpdate
		// TODO
	}
}
<commit_msg>Atom support mostly working (except for href parsing)<commit_after>package main

import (
	"encoding/hex"
	"log"
	"xml"
)

type AtomFeed struct {
	Title string
	Entry []AtomItemData
}

type AtomItemData struct {
	ID      string
	Title   string
	Updated string
	Summary string
}

func readAtom(feed *Feed) {
	url := feed.Info.URL
	log.Print(url)
	r, err := client.Get(url)
	if err != nil {
		log.Print("ERROR: ", err.Error())
	}
	reader := r.Body
	feedData := AtomFeed{}
	err = xml.Unmarshal(reader, &feedData)
	if err != nil {
		log.Print("ERROR: ", err.Error())
		return
	}

	changed := false
	for _, itemData := range feedData.Entry {
		uid := itemData.ID
		id := hex.EncodeToString([]byte(feed.Info.Name + uid))
		_, ok := feed.Items[uid]
		if !ok {
			changed = true
			t := parseTime(itemData.Updated)
			item := &Item {
				Feed: feed,
				Title: itemData.Title,
				GUID: uid,
				URL: "",
				Date: *t,
				Desc: itemData.Summary,
				Read: false,
				ID: id,
			}
			feed.Items[uid] = item
		}
	}

	if (changed) {
		updates <- DoUpdate
		// TODO run some commands from Config?
	}
}
<|endoftext|>


<commit_before>package navitia

import (
	"net/http"
	"time"
)

const (
	// DefaultAPIProtocol is the protocol to be used
	DefaultAPIProtocol = "https"

	// DefaultAPIHostname is the known Navitia API hostname
	DefaultAPIHostname = "api.navitia.io"

	// DefaultAPIVersion is the used API Version
	DefaultAPIVersion = "v1"

	defaultAPIURL = DefaultAPIProtocol + "://" + DefaultAPIHostname + "/" + DefaultAPIVersion

	// Maximum size of response in bytes
	// 10 megabytes
	maxSize int64 = 10 * (1000 * 1000)
)

var defaultClient = &http.Client{}

// Session holds a current session, it is thread-safe
type Session struct {
	APIKey string
	APIURL string

	client  *http.Client
	created time.Time
}

// New creates a new session given an API Key.
// It acts as a convenience wrapper to NewCustom.
//
// Warning: No Timeout is indicated in the default http client, and as such, it is strongly advised to use NewCustom with a custom *http.Client !
func New(key string) (*Session, error) {
	return NewCustom(key, defaultAPIURL, defaultClient)
}

// NewCustom creates a custom new session given an API key, URL to api base & http client
func NewCustom(key string, url string, client *http.Client) (*Session, error) {
	return &Session{
		APIKey:  key,
		APIURL:  url,
		created: time.Now(),
		client:  client,
	}, nil
}
<commit_msg>Add cleaning to NewCustom.<commit_after>package navitia

import (
	"net/http"
	"path"
	"time"
)

const (
	// DefaultAPIProtocol is the protocol to be used
	DefaultAPIProtocol = "https"

	// DefaultAPIHostname is the known Navitia API hostname
	DefaultAPIHostname = "api.navitia.io"

	// DefaultAPIVersion is the used API Version
	DefaultAPIVersion = "v1"

	defaultAPIURL = DefaultAPIProtocol + "://" + DefaultAPIHostname + "/" + DefaultAPIVersion

	// Maximum size of response in bytes
	// 10 megabytes
	maxSize int64 = 10 * (1000 * 1000)
)

var defaultClient = &http.Client{}

// Session holds a current session, it is thread-safe
type Session struct {
	APIKey string
	APIURL string

	client  *http.Client
	created time.Time
}

// New creates a new session given an API Key.
// It acts as a convenience wrapper to NewCustom.
//
// Warning: No Timeout is indicated in the default http client, and as such, it is strongly advised to use NewCustom with a custom *http.Client !
func New(key string) (*Session, error) {
	return NewCustom(key, path.Clean(defaultAPIURL), defaultClient)
}

// NewCustom creates a custom new session given an API key, URL to api base & http client
func NewCustom(key string, url string, client *http.Client) (*Session, error) {
	return &Session{
		APIKey:  key,
		APIURL:  url,
		created: time.Now(),
		client:  client,
	}, nil
}
<|endoftext|>


<commit_before>// demonstrate error handling inside defered anonymous function

// example inspired from https://blevesearch.com/news/Deferred-Cleanup,-Checking-Errors,-and-Potential-Problems/

package main

import (
	"fmt"
	"log"
)

func main() {
	r, err := Open("a")
	if err != nil {
		log.Fatalf("error opening 'a'\n")
	}
	// defer r.Close()

	defer func() {
		err = r.Close()
		if err != nil {
			log.Fatal(err)
		}
	}()

	err = fmt.Errorf("some random error")
	if err != nil {
		log.Printf("received error %v", err)
	}
}

type Resource struct {
	name string
}

func Open(name string) (*Resource, error) {
	return &Resource{name}, nil
}

func (r *Resource) Close() error {
	log.Printf("closing %s\n", r.name)
	return fmt.Errorf("failed to close")
}
<commit_msg>Create with-error-handling.go<commit_after>// demonstrate error handling inside defered anonymous function

// example inspired from https://blevesearch.com/news/Deferred-Cleanup,-Checking-Errors,-and-Potential-Problems/

package main

import (
	"fmt"
	"log"
)

func main() {
	r, err := Open("a")
	if err != nil {
		log.Fatalf("error opening 'a'\n")
	}
	// defer r.Close()

	defer func() {
		err = r.Close()
		if err != nil {
			log.Fatal(err)
		}
	}()

	err = fmt.Errorf("some random error")
	if err != nil {
		log.Printf("received error %v", err)
	}
}

type Resource struct {
	name string
}

func Open(name string) (*Resource, error) {
	return &Resource{name}, nil
}

func (r *Resource) Close() error {
	log.Printf("closing %s\n", r.name)
	return fmt.Errorf("failed to close")
}
<|endoftext|>


<commit_before>package expr

import (
	"math"
	"strconv"
	"testing"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/test"
	schema "gopkg.in/raintank/schema.v1"
)

func TestSeriesAggregatorsIdentity(t *testing.T) {
	input := []models.Series{
		{
			QueryPatt:  "single",
			Datapoints: getCopy(a),
		},
	}
	zeroOutput := getCopy(a)
	for i := range zeroOutput {
		if !math.IsNaN(zeroOutput[i].Val) {
			zeroOutput[i].Val = 0
		}
	}
	testSeriesAggregate("identity", "average", input, getCopy(a), t)
	testSeriesAggregate("identity", "sum", input, getCopy(a), t)
	testSeriesAggregate("identity", "max", input, getCopy(a), t)
	testSeriesAggregate("identity", "min", input, getCopy(a), t)
	testSeriesAggregate("identity", "multiply", input, getCopy(a), t)
	testSeriesAggregate("identity", "median", input, getCopy(a), t)
	testSeriesAggregate("identity", "diff", input, getCopy(a), t)
	testSeriesAggregate("identity", "stddev", input, zeroOutput, t)
	testSeriesAggregate("identity", "rangeOf", input, zeroOutput, t)
	testSeriesAggregate("identity", "range", input, zeroOutput, t)
}

func TestSeriesAggregate2series(t *testing.T) {
	input := []models.Series{
		{
			QueryPatt:  "foo.*",
			Datapoints: getCopy(a),
		},
		{
			QueryPatt:  "foo.*",
			Datapoints: getCopy(b),
		},
	}

	testSeriesAggregate("2Series", "average", input, getCopy(avgab), t)
	testSeriesAggregate("2Series", "sum", input, getCopy(sumab), t)
	testSeriesAggregate("2Series", "max", input, getCopy(maxab), t)
	testSeriesAggregate("2Series", "min", input, getCopy(minab), t)
	testSeriesAggregate("2Series", "multiply", input, getCopy(multab), t)
	testSeriesAggregate("2Series", "median", input, getCopy(medianab), t)
	testSeriesAggregate("2Series", "diff", input, getCopy(diffab), t)
	testSeriesAggregate("2Series", "stddev", input, getCopy(stddevab), t)
	testSeriesAggregate("2Series", "range", input, getCopy(rangeab), t)
}

func TestSeriesAggregate3series(t *testing.T) {
	input := []models.Series{
		{
			QueryPatt:  "foo.*",
			Datapoints: getCopy(a),
		},
		{
			QueryPatt:  "foo.*",
			Datapoints: getCopy(b),
		},
		{
			QueryPatt:  "movingAverage(bar, '1min')",
			Datapoints: getCopy(c),
		},
	}

	testSeriesAggregate("3Series", "average", input, getCopy(avgabc), t)
	testSeriesAggregate("3Series", "sum", input, getCopy(sumabc), t)
	testSeriesAggregate("3Series", "max", input, getCopy(maxabc), t)
	testSeriesAggregate("3Series", "min", input, getCopy(minabc), t)
	testSeriesAggregate("3Series", "multiply", input, getCopy(multabc), t)
	testSeriesAggregate("3Series", "median", input, getCopy(medianabc), t)
	testSeriesAggregate("3Series", "diff", input, getCopy(diffabc), t)
	testSeriesAggregate("3Series", "stddev", input, getCopy(stddevabc), t)
	testSeriesAggregate("3Series", "range", input, getCopy(rangeabc), t)
}

func testSeriesAggregate(name, agg string, in []models.Series, out []schema.Point, t *testing.T) {
	f := getCrossSeriesAggFunc(agg)

	got := make([]schema.Point, 0, len(out))
	f(in, &got)

	if len(got) != len(out) {
		t.Fatalf("case %q (%q): len output expected %d, got %d", name, agg, len(out), len(got))
	}

	// Use EPSILON to avoid floating-point precision errors
	EPSILON := math.Nextafter(1, 2) - 1

	for j, p := range got {
		bothNaN := math.IsNaN(p.Val) && math.IsNaN(out[j].Val)
		if (bothNaN || p.Val == out[j].Val || math.Abs(p.Val-out[j].Val) < EPSILON) && p.Ts == out[j].Ts {
			continue
		}
		t.Fatalf("case %q (%q): output point %d - expected %v got %v", name, agg, j, out[j], p)
	}
}

func BenchmarkSeriesAggregateAvg10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesAvg, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateAvg10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesAvg, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateSum10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesSum, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateSum10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesSum, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateMax10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMax, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateMax10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMax, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateMin10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMin, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateMin10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMin, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateMultiply10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMultiply, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateMultiply10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMultiply, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateMedian10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMedian, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateMedian10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMedian, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateDiff10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesDiff, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateDiff10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesDiff, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateStddev10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesStddev, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateStddev10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesStddev, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateRange10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesRange, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateRange10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesRange, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func benchmarkSeriesAggregate(b *testing.B, aggFunc crossSeriesAggFunc, numSeries int, fn0, fn1 func() []schema.Point) {
	var input []models.Series
	for i := 0; i < numSeries; i++ {
		series := models.Series{
			Target: strconv.Itoa(i),
		}
		if i%1 == 0 {
			series.Datapoints = fn0()
		} else {
			series.Datapoints = fn1()
		}
		input = append(input, series)
	}
	b.ResetTimer()
	out := make([]schema.Point, 0, len(input[0].Datapoints))
	for i := 0; i < b.N; i++ {
		aggFunc(input, &out)
		out = out[:0]
	}
	b.SetBytes(int64(numSeries * len(input[0].Datapoints) * 12))
}
<commit_msg>Remove unnecessary epsilon check<commit_after>package expr

import (
	"math"
	"strconv"
	"testing"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/test"
	schema "gopkg.in/raintank/schema.v1"
)

func TestSeriesAggregatorsIdentity(t *testing.T) {
	input := []models.Series{
		{
			QueryPatt:  "single",
			Datapoints: getCopy(a),
		},
	}
	zeroOutput := getCopy(a)
	for i := range zeroOutput {
		if !math.IsNaN(zeroOutput[i].Val) {
			zeroOutput[i].Val = 0
		}
	}
	testSeriesAggregate("identity", "average", input, getCopy(a), t)
	testSeriesAggregate("identity", "sum", input, getCopy(a), t)
	testSeriesAggregate("identity", "max", input, getCopy(a), t)
	testSeriesAggregate("identity", "min", input, getCopy(a), t)
	testSeriesAggregate("identity", "multiply", input, getCopy(a), t)
	testSeriesAggregate("identity", "median", input, getCopy(a), t)
	testSeriesAggregate("identity", "diff", input, getCopy(a), t)
	testSeriesAggregate("identity", "stddev", input, zeroOutput, t)
	testSeriesAggregate("identity", "rangeOf", input, zeroOutput, t)
	testSeriesAggregate("identity", "range", input, zeroOutput, t)
}

func TestSeriesAggregate2series(t *testing.T) {
	input := []models.Series{
		{
			QueryPatt:  "foo.*",
			Datapoints: getCopy(a),
		},
		{
			QueryPatt:  "foo.*",
			Datapoints: getCopy(b),
		},
	}

	testSeriesAggregate("2Series", "average", input, getCopy(avgab), t)
	testSeriesAggregate("2Series", "sum", input, getCopy(sumab), t)
	testSeriesAggregate("2Series", "max", input, getCopy(maxab), t)
	testSeriesAggregate("2Series", "min", input, getCopy(minab), t)
	testSeriesAggregate("2Series", "multiply", input, getCopy(multab), t)
	testSeriesAggregate("2Series", "median", input, getCopy(medianab), t)
	testSeriesAggregate("2Series", "diff", input, getCopy(diffab), t)
	testSeriesAggregate("2Series", "stddev", input, getCopy(stddevab), t)
	testSeriesAggregate("2Series", "range", input, getCopy(rangeab), t)
}

func TestSeriesAggregate3series(t *testing.T) {
	input := []models.Series{
		{
			QueryPatt:  "foo.*",
			Datapoints: getCopy(a),
		},
		{
			QueryPatt:  "foo.*",
			Datapoints: getCopy(b),
		},
		{
			QueryPatt:  "movingAverage(bar, '1min')",
			Datapoints: getCopy(c),
		},
	}

	testSeriesAggregate("3Series", "average", input, getCopy(avgabc), t)
	testSeriesAggregate("3Series", "sum", input, getCopy(sumabc), t)
	testSeriesAggregate("3Series", "max", input, getCopy(maxabc), t)
	testSeriesAggregate("3Series", "min", input, getCopy(minabc), t)
	testSeriesAggregate("3Series", "multiply", input, getCopy(multabc), t)
	testSeriesAggregate("3Series", "median", input, getCopy(medianabc), t)
	testSeriesAggregate("3Series", "diff", input, getCopy(diffabc), t)
	testSeriesAggregate("3Series", "stddev", input, getCopy(stddevabc), t)
	testSeriesAggregate("3Series", "range", input, getCopy(rangeabc), t)
}

func testSeriesAggregate(name, agg string, in []models.Series, out []schema.Point, t *testing.T) {
	f := getCrossSeriesAggFunc(agg)

	got := make([]schema.Point, 0, len(out))
	f(in, &got)

	if len(got) != len(out) {
		t.Fatalf("case %q (%q): len output expected %d, got %d", name, agg, len(out), len(got))
	}

	for j, p := range got {
		bothNaN := math.IsNaN(p.Val) && math.IsNaN(out[j].Val)
		if (bothNaN || p.Val == out[j].Val) && p.Ts == out[j].Ts {
			continue
		}
		t.Fatalf("case %q (%q): output point %d - expected %v got %v", name, agg, j, out[j], p)
	}
}

func BenchmarkSeriesAggregateAvg10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesAvg, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateAvg10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesAvg, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateSum10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesSum, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateSum10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesSum, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateMax10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMax, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateMax10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMax, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateMin10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMin, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateMin10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMin, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateMultiply10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMultiply, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateMultiply10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMultiply, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateMedian10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMedian, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateMedian10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesMedian, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateDiff10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesDiff, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateDiff10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesDiff, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateStddev10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesStddev, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateStddev10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesStddev, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func BenchmarkSeriesAggregateRange10k_100NoNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesRange, 100, test.RandFloats10k, test.RandFloats10k)
}
func BenchmarkSeriesAggregateRange10k_100WithNulls(b *testing.B) {
	benchmarkSeriesAggregate(b, crossSeriesRange, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
}

func benchmarkSeriesAggregate(b *testing.B, aggFunc crossSeriesAggFunc, numSeries int, fn0, fn1 func() []schema.Point) {
	var input []models.Series
	for i := 0; i < numSeries; i++ {
		series := models.Series{
			Target: strconv.Itoa(i),
		}
		if i%1 == 0 {
			series.Datapoints = fn0()
		} else {
			series.Datapoints = fn1()
		}
		input = append(input, series)
	}
	b.ResetTimer()
	out := make([]schema.Point, 0, len(input[0].Datapoints))
	for i := 0; i < b.N; i++ {
		aggFunc(input, &out)
		out = out[:0]
	}
	b.SetBytes(int64(numSeries * len(input[0].Datapoints) * 12))
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package integration

import (
	"fmt"
	"math/rand"
	"net"
	"strconv"
	"time"

	"github.com/Shopify/sarama"
	"github.com/pborman/uuid"
	cheramiclient "github.com/uber/cherami-client-go/client/cherami"
	"github.com/uber/cherami-server/common"
	"github.com/uber/cherami-thrift/.generated/go/cherami"
)

// Note: you need to start ZooKeeper/Kafka on your local machine to run following tests.
// If running on Mac and java 1.7 for ZooKeeper/Kafka, run following command before starting Kafka:
// echo "127.0.0.1 $HOSTNAME" | sudo tee -a /etc/hosts

const (
	numMsgs          = 1000      // number of messages to send/receive
	minSize, maxSize = 512, 8192 // range for message size

	nKafkaTopics = 5 // number of kafka topics
	kafkaBroker  = "localhost:9092"
	kafkaCluster = "local"
)

type kafkaMsg struct {
	topic string
	key   string
	val   []byte
	part  int32
	offs  int64
	seq   int
}

func (t *kafkaMsg) Equals(m *kafkaMsg) bool {

	if m.topic != t.topic || m.key != t.key ||
		len(m.val) != len(t.val) ||
		m.part != t.part || m.offs != t.offs {

		return false
	}

	for i, b := range m.val {

		if t.val[i] != b {
			return false
		}
	}

	return true
}

func (t *kafkaMsg) String() string {
	return fmt.Sprintf("[%d] (topic:%v key:%v val:%d bytes) => (part:%d, offs:%d)", t.seq, t.topic, t.key, len(t.val), t.part, t.offs)
}

func (s *NetIntegrationSuiteParallelE) TestKafkaForCherami() {

	destPath, cgPath := "/kafka_test_dest/kfc", "/kafka_test_cg/kfc"

	// initialize set of test kafka topics
	kafkaTopics := make([]string, nKafkaTopics)
	for i := range kafkaTopics {
		kafkaTopics[i] = uuid.New()
	}

	// create cherami client
	ipaddr, port, _ := net.SplitHostPort(s.GetFrontend().GetTChannel().PeerInfo().HostPort)
	portNum, _ := strconv.Atoi(port)
	cheramiClient := createCheramiClient("cherami-test-kfc-integration", ipaddr, portNum, nil)

	// create cherami kfc destination
	destDesc, err := cheramiClient.CreateDestination(&cherami.CreateDestinationRequest{
		Path: common.StringPtr(destPath),
		Type: cherami.DestinationTypePtr(cherami.DestinationType_KAFKA),
		ConsumedMessagesRetention:   common.Int32Ptr(60),
		UnconsumedMessagesRetention: common.Int32Ptr(120),
		ChecksumOption:              cherami.ChecksumOption_CRC32IEEE,
		OwnerEmail:                  common.StringPtr("cherami-test-kfc-integration@uber.com"),
		IsMultiZone:                 common.BoolPtr(false),
		KafkaCluster:                common.StringPtr(kafkaCluster),
		KafkaTopics:                 kafkaTopics,
	})
	s.NotNil(destDesc)
	s.NoError(err)

	// create cherami kfc consumer group
	cgDesc, err := cheramiClient.CreateConsumerGroup(&cherami.CreateConsumerGroupRequest{
		ConsumerGroupName:    common.StringPtr(cgPath),
		DestinationPath:      common.StringPtr(destPath),
		LockTimeoutInSeconds: common.Int32Ptr(30),
		MaxDeliveryCount:     common.Int32Ptr(1),
		OwnerEmail:           common.StringPtr("cherami-test-kfc-integration@uber.com"),
	})
	s.NoError(err)
	s.NotNil(cgDesc)

	// setup cherami consumer
	cheramiConsumer := cheramiClient.CreateConsumer(&cheramiclient.CreateConsumerRequest{
		Path:              destPath,
		ConsumerGroupName: cgPath,
		ConsumerName:      "KfCIntegration",
		PrefetchCount:     1,
		Options:           &cheramiclient.ClientOptions{Timeout: time.Second * 30}, // this is the thrift context timeout
	})
	s.NotNil(cheramiConsumer)
	defer cheramiConsumer.Close()

	cheramiMsgsCh, err := cheramiConsumer.Open(make(chan cheramiclient.Delivery, 1))
	s.NoError(err)

	// setup kafka producer
	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForAll
	config.Producer.Return.Successes = true

	kafkaProducer, err := sarama.NewSyncProducer([]string{kafkaBroker}, config)
	s.NoError(err)
	defer kafkaProducer.Close()

	msgs := make(map[string]*kafkaMsg)

	// publish messages to kafka
	for i := 0; i < numMsgs; i++ {

		var topic = kafkaTopics[rand.Intn(len(kafkaTopics))]       // pick one of the topics at random
		var key = uuid.New()                                       // random key
		var val = make([]byte, minSize+rand.Intn(maxSize-minSize)) // random buf
		rand.Read(val)                                             // fill 'val' with random bytes

		part, offs, err :=
			kafkaProducer.SendMessage(
				&sarama.ProducerMessage{
					Topic: topic,
					Key:   sarama.StringEncoder(key),
					Value: sarama.ByteEncoder(val),
				},
			)

		if err != nil {
			fmt.Printf("kafkaProducer.SendMessage (seq=%d topic=%v key=%v val=%d bytes) failed: %v\n", i, topic, key, len(val), err)
			time.Sleep(100 * time.Millisecond)
			i--
			continue
		}

		msgs[key] = &kafkaMsg{topic: topic, key: key, val: val, part: part, offs: offs, seq: i}
	}

	// consume messages from cherami
loop:
	for i := 0; i < numMsgs; i++ {

		select {
		case cmsg := <-cheramiMsgsCh:
			payload := cmsg.GetMessage().Payload
			uc := payload.GetUserContext()

			key, topic := uc[`key`], uc[`topic`]
			part, _ := strconv.Atoi(uc[`partition`])
			offs, _ := strconv.Atoi(uc[`offset`])

			msg := &kafkaMsg{
				topic: topic,
				key:   key,
				val:   payload.GetData(),
				part:  int32(part),
				offs:  int64(offs),
				seq:   i,
			}

			// validate that message is as expected
			if !msg.Equals(msgs[key]) {
				fmt.Printf("received=%v (expected=%v)\n", msg, msgs[key])
				s.Fail("message validation failed")
			}

			delete(msgs, key) // ensure we don't see duplicates

			cmsg.Ack()

		case <-time.After(45 * time.Second):
			s.Fail("cherami-consumer: timed out")
			break loop
		}
	}

	return
}
<commit_msg>fix/debug panic from kfc integration test (#184)<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package integration

import (
	"fmt"
	"math/rand"
	"net"
	"strconv"
	"time"

	"github.com/Shopify/sarama"
	"github.com/pborman/uuid"
	cheramiclient "github.com/uber/cherami-client-go/client/cherami"
	"github.com/uber/cherami-server/common"
	"github.com/uber/cherami-thrift/.generated/go/cherami"
)

// Note: you need to start ZooKeeper/Kafka on your local machine to run following tests.
// If running on Mac and java 1.7 for ZooKeeper/Kafka, run following command before starting Kafka:
// echo "127.0.0.1 $HOSTNAME" | sudo tee -a /etc/hosts

const (
	numMsgs          = 1000      // number of messages to send/receive
	minSize, maxSize = 512, 8192 // range for message size

	nKafkaTopics = 5 // number of kafka topics
	kafkaBroker  = "localhost:9092"
	kafkaCluster = "local"
)

type kafkaMsg struct {
	topic string
	key   string
	val   []byte
	part  int32
	offs  int64
	seq   int
	count int
}

func (t *kafkaMsg) Equals(m *kafkaMsg) bool {

	if m.topic != t.topic || m.key != t.key ||
		len(m.val) != len(t.val) ||
		m.part != t.part || m.offs != t.offs {

		return false
	}

	for i, b := range m.val {

		if t.val[i] != b {
			return false
		}
	}

	return true
}

func (t *kafkaMsg) String() string {
	return fmt.Sprintf("[%d] (topic:%v key:%v val:%d bytes) => (part:%d, offs:%d) <count=%d>",
		t.seq, t.topic, t.key, len(t.val), t.part, t.offs, t.count)
}

func (s *NetIntegrationSuiteParallelE) TestKafkaForCherami() {

	destPath, cgPath := "/kafka_test_dest/kfc", "/kafka_test_cg/kfc"

	// initialize set of test kafka topics
	kafkaTopics := make([]string, nKafkaTopics)
	for i := range kafkaTopics {
		kafkaTopics[i] = uuid.New()
	}

	// create cherami client
	ipaddr, port, _ := net.SplitHostPort(s.GetFrontend().GetTChannel().PeerInfo().HostPort)
	portNum, _ := strconv.Atoi(port)
	cheramiClient := createCheramiClient("cherami-test-kfc-integration", ipaddr, portNum, nil)

	// create cherami kfc destination
	destDesc, err := cheramiClient.CreateDestination(&cherami.CreateDestinationRequest{
		Path: common.StringPtr(destPath),
		Type: cherami.DestinationTypePtr(cherami.DestinationType_KAFKA),
		ConsumedMessagesRetention:   common.Int32Ptr(60),
		UnconsumedMessagesRetention: common.Int32Ptr(120),
		ChecksumOption:              cherami.ChecksumOption_CRC32IEEE,
		OwnerEmail:                  common.StringPtr("cherami-test-kfc-integration@uber.com"),
		IsMultiZone:                 common.BoolPtr(false),
		KafkaCluster:                common.StringPtr(kafkaCluster),
		KafkaTopics:                 kafkaTopics,
	})
	s.NotNil(destDesc)
	s.NoError(err)

	// create cherami kfc consumer group
	cgDesc, err := cheramiClient.CreateConsumerGroup(&cherami.CreateConsumerGroupRequest{
		ConsumerGroupName:    common.StringPtr(cgPath),
		DestinationPath:      common.StringPtr(destPath),
		LockTimeoutInSeconds: common.Int32Ptr(30),
		MaxDeliveryCount:     common.Int32Ptr(1),
		OwnerEmail:           common.StringPtr("cherami-test-kfc-integration@uber.com"),
	})
	s.NoError(err)
	s.NotNil(cgDesc)

	// setup cherami consumer
	cheramiConsumer := cheramiClient.CreateConsumer(&cheramiclient.CreateConsumerRequest{
		Path:              destPath,
		ConsumerGroupName: cgPath,
		ConsumerName:      "KfCIntegration",
		PrefetchCount:     1,
		Options:           &cheramiclient.ClientOptions{Timeout: time.Second * 30}, // this is the thrift context timeout
	})
	s.NotNil(cheramiConsumer)
	defer cheramiConsumer.Close()

	cheramiMsgsCh, err := cheramiConsumer.Open(make(chan cheramiclient.Delivery, 1))
	s.NoError(err)

	// setup kafka producer
	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForAll
	config.Producer.Return.Successes = true

	kafkaProducer, err := sarama.NewSyncProducer([]string{kafkaBroker}, config)
	if err != nil {
		fmt.Printf("sarama.NewSyncProducer error=%v\n", err)
	}
	s.NoError(err)
	defer kafkaProducer.Close()

	sentMsgs := make(map[string]*kafkaMsg)

	// publish messages to kafka
	for i := 0; i < numMsgs; i++ {

		var topic = kafkaTopics[rand.Intn(len(kafkaTopics))]       // pick one of the topics at random
		var key = uuid.New()                                       // random key
		var val = make([]byte, minSize+rand.Intn(maxSize-minSize)) // random buf
		rand.Read(val)                                             // fill 'val' with random bytes

		part, offs, err :=
			kafkaProducer.SendMessage(
				&sarama.ProducerMessage{
					Topic: topic,
					Key:   sarama.StringEncoder(key),
					Value: sarama.ByteEncoder(val),
				},
			)

		if err != nil {
			fmt.Printf("kafkaProducer.SendMessage (seq=%d topic=%v key=%v val=%d bytes) failed: %v\n", i, topic, key, len(val), err)
			time.Sleep(100 * time.Millisecond)
			i--
			continue
		}

		sentMsgs[key] = &kafkaMsg{
			topic: topic,
			key:   key,
			val:   val,
			part:  part,
			offs:  offs,
			seq:   i,
		}
	}

	recvMsgs := make(map[string]*kafkaMsg)

	// consume messages from cherami
loop:
	for i := 0; (len(recvMsgs) < numMsgs) && (i < 2*numMsgs); i++ {

		select {
		case cmsg := <-cheramiMsgsCh:
			payload := cmsg.GetMessage().Payload
			uc := payload.GetUserContext()

			key, topic := uc[`key`], uc[`topic`]
			part, _ := strconv.Atoi(uc[`partition`])
			offs, _ := strconv.Atoi(uc[`offset`])

			// check for duplicates
			if _, ok := recvMsgs[key]; ok {

				if recvMsgs[key].count++; recvMsgs[key].count > 3 {
					s.Fail("received message too many times")
				}

				continue loop
			}

			msg := &kafkaMsg{
				topic: topic,
				key:   key,
				val:   payload.GetData(),
				part:  int32(part),
				offs:  int64(offs),
				seq:   i,
				count: 1,
			}

			// validate that message is as expected
			if sentMsgs[key] == nil || !msg.Equals(sentMsgs[key]) {

				fmt.Printf("received[%d]=%v\n", i, msg)

				if sentMsgs[key] == nil {
					fmt.Printf("expected=<MISSING>\n")
				} else {
					fmt.Printf("expected=%v\n", sentMsgs[key])
				}

				fmt.Printf("\nreceived:\n")
				for _, m := range recvMsgs {
					fmt.Printf("%v\n", m)
				}

				fmt.Printf("\nsent (and not received):\n")
				for _, m := range sentMsgs {
					fmt.Printf("%v\n", m)
				}

				s.Fail("message validation failed")
			}

			// if we have seen all the messages, break out
			recvMsgs[key] = msg

			cmsg.Ack()

		case <-time.After(45 * time.Second):
			s.Fail("cherami-consumer: timed out")
			break loop
		}
	}

	s.Equal(numMsgs, len(recvMsgs)) // we should have received all the messages

	return
}
<|endoftext|>


<commit_before>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package gvisor provides support for gVisor, user-space kernel, testing.
// See https://github.com/google/gvisor
package gvisor

import (
	"bytes"
	"fmt"
	"io"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/google/syzkaller/pkg/config"
	"github.com/google/syzkaller/pkg/log"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/vm/vmimpl"
)

func init() {
	vmimpl.Register("gvisor", ctor, true)
}

type Config struct {
	Count     int    `json:"count"` // number of VMs to use
	RunscArgs string `json:"runsc_args"`
}

type Pool struct {
	env *vmimpl.Env
	cfg *Config
}

type instance struct {
	cfg      *Config
	image    string
	debug    bool
	rootDir  string
	imageDir string
	name     string
	port     int
	cmd      *exec.Cmd
	merger   *vmimpl.OutputMerger
}

func ctor(env *vmimpl.Env) (vmimpl.Pool, error) {
	cfg := &Config{
		Count: 1,
	}
	if err := config.LoadData(env.Config, cfg); err != nil {
		return nil, fmt.Errorf("failed to parse vm config: %v", err)
	}
	if cfg.Count < 1 || cfg.Count > 128 {
		return nil, fmt.Errorf("invalid config param count: %v, want [1, 128]", cfg.Count)
	}
	if env.Debug && cfg.Count > 1 {
		log.Logf(0, "limiting number of VMs from %v to 1 in debug mode", cfg.Count)
		cfg.Count = 1
	}
	if !osutil.IsExist(env.Image) {
		return nil, fmt.Errorf("image file %q does not exist", env.Image)
	}
	pool := &Pool{
		cfg: cfg,
		env: env,
	}
	return pool, nil
}

func (pool *Pool) Count() int {
	return pool.cfg.Count
}

func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) {
	rootDir := filepath.Clean(filepath.Join(workdir, "..", "gvisor_root"))
	imageDir := filepath.Join(workdir, "image")
	bundleDir := filepath.Join(workdir, "bundle")
	osutil.MkdirAll(rootDir)
	osutil.MkdirAll(bundleDir)
	osutil.MkdirAll(imageDir)

	caps := ""
	for _, c := range sandboxCaps {
		if caps != "" {
			caps += ", "
		}
		caps += "\"" + c + "\""
	}
	vmConfig := fmt.Sprintf(configTempl, imageDir, caps)
	if err := osutil.WriteFile(filepath.Join(bundleDir, "config.json"), []byte(vmConfig)); err != nil {
		return nil, err
	}
	bin, err := exec.LookPath(os.Args[0])
	if err != nil {
		return nil, fmt.Errorf("failed to lookup %v: %v", os.Args[0], err)
	}
	if err := osutil.CopyFile(bin, filepath.Join(imageDir, "init")); err != nil {
		return nil, err
	}

	panicLog := filepath.Join(bundleDir, "panic.fifo")
	if err := syscall.Mkfifo(panicLog, 0666); err != nil {
		return nil, err
	}
	defer syscall.Unlink(panicLog)

	// Open the fifo for read-write to be able to open for read-only
	// without blocking.
	panicLogWriteFD, err := os.OpenFile(panicLog, os.O_RDWR, 0)
	if err != nil {
		return nil, err
	}
	defer panicLogWriteFD.Close()

	panicLogReadFD, err := os.Open(panicLog)
	if err != nil {
		return nil, err
	}

	rpipe, wpipe, err := osutil.LongPipe()
	if err != nil {
		panicLogReadFD.Close()
		return nil, err
	}
	var tee io.Writer
	if pool.env.Debug {
		tee = os.Stdout
	}
	merger := vmimpl.NewOutputMerger(tee)
	merger.Add("gvisor", rpipe)
	merger.Add("gvisor-goruntime", panicLogReadFD)

	inst := &instance{
		cfg:      pool.cfg,
		image:    pool.env.Image,
		debug:    pool.env.Debug,
		rootDir:  rootDir,
		imageDir: imageDir,
		name:     fmt.Sprintf("%v-%v", pool.env.Name, index),
		merger:   merger,
	}

	// Kill the previous instance in case it's still running.
	osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name))
	time.Sleep(3 * time.Second)

	cmd := inst.runscCmd("--panic-log", panicLog, "run", "-bundle", bundleDir, inst.name)
	cmd.Stdout = wpipe
	cmd.Stderr = wpipe
	if err := cmd.Start(); err != nil {
		wpipe.Close()
		panicLogWriteFD.Close()
		merger.Wait()
		return nil, err
	}
	inst.cmd = cmd
	wpipe.Close()

	if err := inst.waitBoot(); err != nil {
		panicLogWriteFD.Close()
		inst.Close()
		return nil, err
	}
	return inst, nil
}

func (inst *instance) waitBoot() error {
	errorMsg := []byte("FATAL ERROR:")
	bootedMsg := []byte(initStartMsg)
	timeout := time.NewTimer(time.Minute)
	defer timeout.Stop()
	var output []byte
	for {
		select {
		case out := <-inst.merger.Output:
			output = append(output, out...)
			if pos := bytes.Index(output, errorMsg); pos != -1 {
				end := bytes.IndexByte(output[pos:], '\n')
				if end == -1 {
					end = len(output)
				} else {
					end += pos
				}
				return vmimpl.BootError{
					Title:  string(output[pos:end]),
					Output: output,
				}
			}
			if bytes.Contains(output, bootedMsg) {
				return nil
			}
		case err := <-inst.merger.Err:
			return vmimpl.BootError{
				Title:  fmt.Sprintf("runsc failed: %v", err),
				Output: output,
			}
		case <-timeout.C:
			return vmimpl.BootError{
				Title:  "init process did not start",
				Output: output,
			}
		}
	}
}

func (inst *instance) runscCmd(add ...string) *exec.Cmd {
	args := []string{
		"-root", inst.rootDir,
		"-watchdog-action=panic",
		"-network=none",
		"-debug",
		"-alsologtostderr",
	}
	if inst.cfg.RunscArgs != "" {
		args = append(args, strings.Split(inst.cfg.RunscArgs, " ")...)
	}
	args = append(args, add...)
	cmd := osutil.Command(inst.image, args...)
	cmd.Env = []string{
		"GOTRACEBACK=all",
		"GORACE=halt_on_error=1",
	}
	return cmd
}

func (inst *instance) Close() {
	time.Sleep(3 * time.Second)
	osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name))
	inst.cmd.Process.Kill()
	inst.merger.Wait()
	inst.cmd.Wait()
	osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name))
	time.Sleep(3 * time.Second)
}

func (inst *instance) Forward(port int) (string, error) {
	if inst.port != 0 {
		return "", fmt.Errorf("forward port is already setup")
	}
	inst.port = port
	return "stdin", nil
}

func (inst *instance) Copy(hostSrc string) (string, error) {
	fname := filepath.Base(hostSrc)
	if err := osutil.CopyFile(hostSrc, filepath.Join(inst.imageDir, fname)); err != nil {
		return "", err
	}
	if err := os.Chmod(inst.imageDir, 0777); err != nil {
		return "", err
	}
	return filepath.Join("/", fname), nil
}

func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command string) (
	<-chan []byte, <-chan error, error) {
	args := []string{"exec", "-user=0:0"}
	for _, c := range sandboxCaps {
		args = append(args, "-cap", c)
	}
	args = append(args, inst.name)
	args = append(args, strings.Split(command, " ")...)
	cmd := inst.runscCmd(args...)

	rpipe, wpipe, err := osutil.LongPipe()
	if err != nil {
		return nil, nil, err
	}
	defer wpipe.Close()
	inst.merger.Add("cmd", rpipe)
	cmd.Stdout = wpipe
	cmd.Stderr = wpipe

	guestSock, err := inst.guestProxy()
	if err != nil {
		return nil, nil, err
	}
	if guestSock != nil {
		defer guestSock.Close()
		cmd.Stdin = guestSock
	}

	if err := cmd.Start(); err != nil {
		return nil, nil, err
	}
	errc := make(chan error, 1)
	signal := func(err error) {
		select {
		case errc <- err:
		default:
		}
	}

	go func() {
		select {
		case <-time.After(timeout):
			signal(vmimpl.ErrTimeout)
		case <-stop:
			signal(vmimpl.ErrTimeout)
		case err := <-inst.merger.Err:
			cmd.Process.Kill()
			if cmdErr := cmd.Wait(); cmdErr == nil {
				// If the command exited successfully, we got EOF error from merger.
				// But in this case no error has happened and the EOF is expected.
				err = nil
			}
			signal(err)
			return
		}
		cmd.Process.Kill()
		cmd.Wait()
	}()
	return inst.merger.Output, errc, nil
}

func (inst *instance) guestProxy() (*os.File, error) {
	if inst.port == 0 {
		return nil, nil
	}
	// One does not simply let gvisor guest connect to host tcp port.
	// We create a unix socket, pass it to guest in stdin.
	// Guest will use it instead of dialing manager directly.
	// On host we connect to manager tcp port and proxy between the tcp and unix connections.
	socks, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
	if err != nil {
		return nil, err
	}
	hostSock := os.NewFile(uintptr(socks[0]), "host unix proxy")
	guestSock := os.NewFile(uintptr(socks[1]), "guest unix proxy")
	conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%v", inst.port))
	if err != nil {
		hostSock.Close()
		guestSock.Close()
		return nil, err
	}
	go func() {
		io.Copy(hostSock, conn)
		hostSock.Close()
	}()
	go func() {
		io.Copy(conn, hostSock)
		conn.Close()
	}()
	return guestSock, nil
}

func (inst *instance) Diagnose() ([]byte, bool) {
	b, err := osutil.Run(time.Minute, inst.runscCmd("debug", "-stacks", "--ps", inst.name))
	if err != nil {
		b = append(b, fmt.Sprintf("\n\nError collecting stacks: %v", err)...)
	}
	b1, err := osutil.RunCmd(time.Minute, "", "dmesg")
	b = append(b, b1...)
	if err != nil {
		b = append(b, fmt.Sprintf("\n\nError collecting kernel logs: %v", err)...)
	}
	return b, false
}

func init() {
	if os.Getenv("SYZ_GVISOR_PROXY") != "" {
		fmt.Fprint(os.Stderr, initStartMsg)
		// If we do select{}, we can get a deadlock panic.
		for range time.NewTicker(time.Hour).C {
		}
	}
}

const initStartMsg = "SYZKALLER INIT STARTED\n"

const configTempl = `
{
	"root": {
		"path": "%[1]v",
		"readonly": true
	},
	"process":{
                "args": ["/init"],
                "cwd": "/tmp",
                "env": ["SYZ_GVISOR_PROXY=1"],
                "capabilities": {
                	"bounding": [%[2]v],
                	"effective": [%[2]v],
                	"inheritable": [%[2]v],
                	"permitted": [%[2]v],
                	"ambient": [%[2]v]
                }
	}
}
`

var sandboxCaps = []string{
	"CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_DAC_READ_SEARCH", "CAP_FOWNER", "CAP_FSETID",
	"CAP_KILL", "CAP_SETGID", "CAP_SETUID", "CAP_SETPCAP", "CAP_LINUX_IMMUTABLE",
	"CAP_NET_BIND_SERVICE", "CAP_NET_BROADCAST", "CAP_NET_ADMIN", "CAP_NET_RAW",
	"CAP_IPC_LOCK", "CAP_IPC_OWNER", "CAP_SYS_MODULE", "CAP_SYS_RAWIO", "CAP_SYS_CHROOT",
	"CAP_SYS_PTRACE", "CAP_SYS_PACCT", "CAP_SYS_ADMIN", "CAP_SYS_BOOT", "CAP_SYS_NICE",
	"CAP_SYS_RESOURCE", "CAP_SYS_TIME", "CAP_SYS_TTY_CONFIG", "CAP_MKNOD", "CAP_LEASE",
	"CAP_AUDIT_WRITE", "CAP_AUDIT_CONTROL", "CAP_SETFCAP", "CAP_MAC_OVERRIDE", "CAP_MAC_ADMIN",
	"CAP_SYSLOG", "CAP_WAKE_ALARM", "CAP_BLOCK_SUSPEND", "CAP_AUDIT_READ",
}
<commit_msg>vm/gvisor: Send debug logs to stderr so they can be read by syzkaller<commit_after>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package gvisor provides support for gVisor, user-space kernel, testing.
// See https://github.com/google/gvisor
package gvisor

import (
	"bytes"
	"fmt"
	"io"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/google/syzkaller/pkg/config"
	"github.com/google/syzkaller/pkg/log"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/vm/vmimpl"
)

func init() {
	vmimpl.Register("gvisor", ctor, true)
}

type Config struct {
	Count     int    `json:"count"` // number of VMs to use
	RunscArgs string `json:"runsc_args"`
}

type Pool struct {
	env *vmimpl.Env
	cfg *Config
}

type instance struct {
	cfg      *Config
	image    string
	debug    bool
	rootDir  string
	imageDir string
	name     string
	port     int
	cmd      *exec.Cmd
	merger   *vmimpl.OutputMerger
}

func ctor(env *vmimpl.Env) (vmimpl.Pool, error) {
	cfg := &Config{
		Count: 1,
	}
	if err := config.LoadData(env.Config, cfg); err != nil {
		return nil, fmt.Errorf("failed to parse vm config: %v", err)
	}
	if cfg.Count < 1 || cfg.Count > 128 {
		return nil, fmt.Errorf("invalid config param count: %v, want [1, 128]", cfg.Count)
	}
	if env.Debug && cfg.Count > 1 {
		log.Logf(0, "limiting number of VMs from %v to 1 in debug mode", cfg.Count)
		cfg.Count = 1
	}
	if !osutil.IsExist(env.Image) {
		return nil, fmt.Errorf("image file %q does not exist", env.Image)
	}
	pool := &Pool{
		cfg: cfg,
		env: env,
	}
	return pool, nil
}

func (pool *Pool) Count() int {
	return pool.cfg.Count
}

func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) {
	rootDir := filepath.Clean(filepath.Join(workdir, "..", "gvisor_root"))
	imageDir := filepath.Join(workdir, "image")
	bundleDir := filepath.Join(workdir, "bundle")
	osutil.MkdirAll(rootDir)
	osutil.MkdirAll(bundleDir)
	osutil.MkdirAll(imageDir)

	caps := ""
	for _, c := range sandboxCaps {
		if caps != "" {
			caps += ", "
		}
		caps += "\"" + c + "\""
	}
	vmConfig := fmt.Sprintf(configTempl, imageDir, caps)
	if err := osutil.WriteFile(filepath.Join(bundleDir, "config.json"), []byte(vmConfig)); err != nil {
		return nil, err
	}
	bin, err := exec.LookPath(os.Args[0])
	if err != nil {
		return nil, fmt.Errorf("failed to lookup %v: %v", os.Args[0], err)
	}
	if err := osutil.CopyFile(bin, filepath.Join(imageDir, "init")); err != nil {
		return nil, err
	}

	panicLog := filepath.Join(bundleDir, "panic.fifo")
	if err := syscall.Mkfifo(panicLog, 0666); err != nil {
		return nil, err
	}
	defer syscall.Unlink(panicLog)

	// Open the fifo for read-write to be able to open for read-only
	// without blocking.
	panicLogWriteFD, err := os.OpenFile(panicLog, os.O_RDWR, 0)
	if err != nil {
		return nil, err
	}
	defer panicLogWriteFD.Close()

	panicLogReadFD, err := os.Open(panicLog)
	if err != nil {
		return nil, err
	}

	rpipe, wpipe, err := osutil.LongPipe()
	if err != nil {
		panicLogReadFD.Close()
		return nil, err
	}
	var tee io.Writer
	if pool.env.Debug {
		tee = os.Stdout
	}
	merger := vmimpl.NewOutputMerger(tee)
	merger.Add("gvisor", rpipe)
	merger.Add("gvisor-goruntime", panicLogReadFD)

	inst := &instance{
		cfg:      pool.cfg,
		image:    pool.env.Image,
		debug:    pool.env.Debug,
		rootDir:  rootDir,
		imageDir: imageDir,
		name:     fmt.Sprintf("%v-%v", pool.env.Name, index),
		merger:   merger,
	}

	// Kill the previous instance in case it's still running.
	osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name))
	time.Sleep(3 * time.Second)

	cmd := inst.runscCmd("--panic-log", panicLog, "run", "-bundle", bundleDir, inst.name)
	cmd.Stdout = wpipe
	cmd.Stderr = wpipe
	if err := cmd.Start(); err != nil {
		wpipe.Close()
		panicLogWriteFD.Close()
		merger.Wait()
		return nil, err
	}
	inst.cmd = cmd
	wpipe.Close()

	if err := inst.waitBoot(); err != nil {
		panicLogWriteFD.Close()
		inst.Close()
		return nil, err
	}
	return inst, nil
}

func (inst *instance) waitBoot() error {
	errorMsg := []byte("FATAL ERROR:")
	bootedMsg := []byte(initStartMsg)
	timeout := time.NewTimer(time.Minute)
	defer timeout.Stop()
	var output []byte
	for {
		select {
		case out := <-inst.merger.Output:
			output = append(output, out...)
			if pos := bytes.Index(output, errorMsg); pos != -1 {
				end := bytes.IndexByte(output[pos:], '\n')
				if end == -1 {
					end = len(output)
				} else {
					end += pos
				}
				return vmimpl.BootError{
					Title:  string(output[pos:end]),
					Output: output,
				}
			}
			if bytes.Contains(output, bootedMsg) {
				return nil
			}
		case err := <-inst.merger.Err:
			return vmimpl.BootError{
				Title:  fmt.Sprintf("runsc failed: %v", err),
				Output: output,
			}
		case <-timeout.C:
			return vmimpl.BootError{
				Title:  "init process did not start",
				Output: output,
			}
		}
	}
}

func (inst *instance) runscCmd(add ...string) *exec.Cmd {
	args := []string{
		"-root", inst.rootDir,
		"-watchdog-action=panic",
		"-network=none",
		"-debug",
		// Send debug logs to stderr, so that they will be picked up by
		// syzkaller. Without this, debug logs are sent to /dev/null.
		"-debug-log=/dev/stderr",
	}
	if inst.cfg.RunscArgs != "" {
		args = append(args, strings.Split(inst.cfg.RunscArgs, " ")...)
	}
	args = append(args, add...)
	cmd := osutil.Command(inst.image, args...)
	cmd.Env = []string{
		"GOTRACEBACK=all",
		"GORACE=halt_on_error=1",
	}
	return cmd
}

func (inst *instance) Close() {
	time.Sleep(3 * time.Second)
	osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name))
	inst.cmd.Process.Kill()
	inst.merger.Wait()
	inst.cmd.Wait()
	osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name))
	time.Sleep(3 * time.Second)
}

func (inst *instance) Forward(port int) (string, error) {
	if inst.port != 0 {
		return "", fmt.Errorf("forward port is already setup")
	}
	inst.port = port
	return "stdin", nil
}

func (inst *instance) Copy(hostSrc string) (string, error) {
	fname := filepath.Base(hostSrc)
	if err := osutil.CopyFile(hostSrc, filepath.Join(inst.imageDir, fname)); err != nil {
		return "", err
	}
	if err := os.Chmod(inst.imageDir, 0777); err != nil {
		return "", err
	}
	return filepath.Join("/", fname), nil
}

func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command string) (
	<-chan []byte, <-chan error, error) {
	args := []string{"exec", "-user=0:0"}
	for _, c := range sandboxCaps {
		args = append(args, "-cap", c)
	}
	args = append(args, inst.name)
	args = append(args, strings.Split(command, " ")...)
	cmd := inst.runscCmd(args...)

	rpipe, wpipe, err := osutil.LongPipe()
	if err != nil {
		return nil, nil, err
	}
	defer wpipe.Close()
	inst.merger.Add("cmd", rpipe)
	cmd.Stdout = wpipe
	cmd.Stderr = wpipe

	guestSock, err := inst.guestProxy()
	if err != nil {
		return nil, nil, err
	}
	if guestSock != nil {
		defer guestSock.Close()
		cmd.Stdin = guestSock
	}

	if err := cmd.Start(); err != nil {
		return nil, nil, err
	}
	errc := make(chan error, 1)
	signal := func(err error) {
		select {
		case errc <- err:
		default:
		}
	}

	go func() {
		select {
		case <-time.After(timeout):
			signal(vmimpl.ErrTimeout)
		case <-stop:
			signal(vmimpl.ErrTimeout)
		case err := <-inst.merger.Err:
			cmd.Process.Kill()
			if cmdErr := cmd.Wait(); cmdErr == nil {
				// If the command exited successfully, we got EOF error from merger.
				// But in this case no error has happened and the EOF is expected.
				err = nil
			}
			signal(err)
			return
		}
		cmd.Process.Kill()
		cmd.Wait()
	}()
	return inst.merger.Output, errc, nil
}

func (inst *instance) guestProxy() (*os.File, error) {
	if inst.port == 0 {
		return nil, nil
	}
	// One does not simply let gvisor guest connect to host tcp port.
	// We create a unix socket, pass it to guest in stdin.
	// Guest will use it instead of dialing manager directly.
	// On host we connect to manager tcp port and proxy between the tcp and unix connections.
	socks, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
	if err != nil {
		return nil, err
	}
	hostSock := os.NewFile(uintptr(socks[0]), "host unix proxy")
	guestSock := os.NewFile(uintptr(socks[1]), "guest unix proxy")
	conn, err := net.Dial("tcp", fmt.Sprintf("localhost:%v", inst.port))
	if err != nil {
		hostSock.Close()
		guestSock.Close()
		return nil, err
	}
	go func() {
		io.Copy(hostSock, conn)
		hostSock.Close()
	}()
	go func() {
		io.Copy(conn, hostSock)
		conn.Close()
	}()
	return guestSock, nil
}

func (inst *instance) Diagnose() ([]byte, bool) {
	b, err := osutil.Run(time.Minute, inst.runscCmd("debug", "-stacks", "--ps", inst.name))
	if err != nil {
		b = append(b, fmt.Sprintf("\n\nError collecting stacks: %v", err)...)
	}
	b1, err := osutil.RunCmd(time.Minute, "", "dmesg")
	b = append(b, b1...)
	if err != nil {
		b = append(b, fmt.Sprintf("\n\nError collecting kernel logs: %v", err)...)
	}
	return b, false
}

func init() {
	if os.Getenv("SYZ_GVISOR_PROXY") != "" {
		fmt.Fprint(os.Stderr, initStartMsg)
		// If we do select{}, we can get a deadlock panic.
		for range time.NewTicker(time.Hour).C {
		}
	}
}

const initStartMsg = "SYZKALLER INIT STARTED\n"

const configTempl = `
{
	"root": {
		"path": "%[1]v",
		"readonly": true
	},
	"process":{
                "args": ["/init"],
                "cwd": "/tmp",
                "env": ["SYZ_GVISOR_PROXY=1"],
                "capabilities": {
                	"bounding": [%[2]v],
                	"effective": [%[2]v],
                	"inheritable": [%[2]v],
                	"permitted": [%[2]v],
                	"ambient": [%[2]v]
                }
	}
}
`

var sandboxCaps = []string{
	"CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_DAC_READ_SEARCH", "CAP_FOWNER", "CAP_FSETID",
	"CAP_KILL", "CAP_SETGID", "CAP_SETUID", "CAP_SETPCAP", "CAP_LINUX_IMMUTABLE",
	"CAP_NET_BIND_SERVICE", "CAP_NET_BROADCAST", "CAP_NET_ADMIN", "CAP_NET_RAW",
	"CAP_IPC_LOCK", "CAP_IPC_OWNER", "CAP_SYS_MODULE", "CAP_SYS_RAWIO", "CAP_SYS_CHROOT",
	"CAP_SYS_PTRACE", "CAP_SYS_PACCT", "CAP_SYS_ADMIN", "CAP_SYS_BOOT", "CAP_SYS_NICE",
	"CAP_SYS_RESOURCE", "CAP_SYS_TIME", "CAP_SYS_TTY_CONFIG", "CAP_MKNOD", "CAP_LEASE",
	"CAP_AUDIT_WRITE", "CAP_AUDIT_CONTROL", "CAP_SETFCAP", "CAP_MAC_OVERRIDE", "CAP_MAC_ADMIN",
	"CAP_SYSLOG", "CAP_WAKE_ALARM", "CAP_BLOCK_SUSPEND", "CAP_AUDIT_READ",
}
<|endoftext|>


<commit_before>/**
 * Copyright 2017 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package volume

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"

	k8sresources "github.com/IBM/ubiquity-k8s/resources"
	"github.com/IBM/ubiquity/resources"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/client-go/pkg/api/v1"
)

const (

	// are we allowed to set this? else make up our own
	annCreatedBy = "kubernetes.io/createdby"
	createdBy    = k8sresources.UbiquityProvisionerName

	// Name of the file where an nfsProvisioner will store its identity
	identityFile = "k8sresources.UbiquityProvisionerName" + ".identity"

	// VolumeGidAnnotationKey is the key of the annotation on the PersistentVolume
	// object that specifies a supplemental GID.
	VolumeGidAnnotationKey = "pv.beta.kubernetes.io/gid"

	// A PV annotation for the identity of the flexProvisioner that provisioned it
	annProvisionerId = "Provisioner_Id"

	podIPEnv     = "POD_IP"
	serviceEnv   = "SERVICE_NAME"
	namespaceEnv = "POD_NAMESPACE"
	nodeEnv      = "NODE_NAME"
)

func NewFlexProvisioner(logger *log.Logger, ubiquityClient resources.StorageClient, config resources.UbiquityPluginConfig) (controller.Provisioner, error) {
	return newFlexProvisionerInternal(logger, ubiquityClient, config)
}

func newFlexProvisionerInternal(logger *log.Logger, ubiquityClient resources.StorageClient, config resources.UbiquityPluginConfig) (*flexProvisioner, error) {
	var identity types.UID
	identityPath := path.Join(config.LogPath, identityFile)
	if _, err := os.Stat(identityPath); os.IsNotExist(err) {
		identity = uuid.NewUUID()
		err := ioutil.WriteFile(identityPath, []byte(identity), 0600)
		if err != nil {
			logger.Printf("Error writing identity file %s! %v", identityPath, err)
		}
	} else {
		read, err := ioutil.ReadFile(identityPath)
		if err != nil {
			logger.Printf("Error reading identity file %s! %v", config.LogPath, err)
		}
		identity = types.UID(strings.TrimSpace(string(read)))
	}
	provisioner := &flexProvisioner{
		logger:         logger,
		identity:       identity,
		ubiquityClient: ubiquityClient,
		ubiquityConfig: config,
		podIPEnv:       podIPEnv,
		serviceEnv:     serviceEnv,
		namespaceEnv:   namespaceEnv,
		nodeEnv:        nodeEnv,
	}

	activateRequest := resources.ActivateRequest{Backends: config.Backends}
	err := provisioner.ubiquityClient.Activate(activateRequest)

	return provisioner, err
}

type flexProvisioner struct {
	logger   *log.Logger
	identity types.UID
	// Whether the provisioner is running out of cluster and so cannot rely on
	// the existence of any of the pod, service, namespace, node env variables.
	outOfCluster bool

	ubiquityClient resources.StorageClient
	ubiquityConfig resources.UbiquityPluginConfig

	// Environment variables the provisioner pod needs valid values for in order to
	// put a service cluster IP as the server of provisioned NFS PVs, passed in
	// via downward API. If serviceEnv is set, namespaceEnv must be too.
	podIPEnv     string
	serviceEnv   string
	namespaceEnv string
	nodeEnv      string
}

// Provision creates a volume i.e. the storage asset and returns a PV object for
// the volume.
func (p *flexProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) {
	if options.PVC == nil {
		return nil, fmt.Errorf("options missing PVC %#v", options)
	}

	// change volume name to match the PVC name
	options.PVName = options.PVC.Namespace + "-" + options.PVC.Name

	capacity, exists := options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	if !exists {
		return nil, fmt.Errorf("options.PVC.Spec.Resources.Requests does not contain capacity")
	}
	fmt.Printf("PVC with capacity %d", capacity.Value())
	capacityMB := capacity.Value() / (1024 * 1024)

	volume_details, err := p.createVolume(options, capacityMB)
	if err != nil {
		return nil, err
	}

	annotations := make(map[string]string)
	annotations[annCreatedBy] = createdBy
	annotations[annProvisionerId] = k8sresources.UbiquityProvisionerName

	pv := &v1.PersistentVolume{
		ObjectMeta: v1.ObjectMeta{
			Name:        options.PVName,
			Labels:      map[string]string{},
			Annotations: annotations,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
			AccessModes:                   options.PVC.Spec.AccessModes,
			Capacity: v1.ResourceList{
				v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)],
			},
			PersistentVolumeSource: v1.PersistentVolumeSource{
				FlexVolume: &v1.FlexVolumeSource{
					Driver:    k8sresources.UbiquityK8sFlexVolumeDriverFullName,
					FSType:    "",
					SecretRef: nil,
					ReadOnly:  false,
					Options:   volume_details,
				},
			},
		},
	}

	return pv, nil
}

// Delete removes the directory that was created by Provision backing the given
// PV.
func (p *flexProvisioner) Delete(volume *v1.PersistentVolume) error {
	if volume.Name == "" {
		return fmt.Errorf("volume name cannot be empty %#v", volume)
	}

	if volume.Spec.PersistentVolumeReclaimPolicy != v1.PersistentVolumeReclaimRetain {
		getVolumeRequest := resources.GetVolumeRequest{Name: volume.Name}
		volume, err := p.ubiquityClient.GetVolume(getVolumeRequest)
		if err != nil {
			fmt.Printf("error-retrieving-volume-info")
			return err
		}
		removeVolumeRequest := resources.RemoveVolumeRequest{Name: volume.Name}
		err = p.ubiquityClient.RemoveVolume(removeVolumeRequest)
		if err != nil {
			return err
		}
		return nil

	}

	return nil
}

func (p *flexProvisioner) createVolume(options controller.VolumeOptions, capacity int64) (map[string]string, error) {
	ubiquityParams := make(map[string]interface{})
	if capacity != 0 {
		ubiquityParams["quota"] = fmt.Sprintf("%dM", capacity)    // SSc backend expect quota option
		ubiquityParams["size"] = fmt.Sprintf("%d", capacity/1024) // SCBE backend expect size option
	}
	for key, value := range options.Parameters {
		ubiquityParams[key] = value
	}
	backendName, exists := ubiquityParams["backend"]
	if !exists {
		return nil, fmt.Errorf("backend is not specified")
	}
	b := backendName.(string)
	createVolumeRequest := resources.CreateVolumeRequest{Name: options.PVName, Backend: b, Opts: ubiquityParams}
	err := p.ubiquityClient.CreateVolume(createVolumeRequest)
	if err != nil {
		return nil, fmt.Errorf("error creating volume: %v", err)
	}

	getVolumeConfigRequest := resources.GetVolumeConfigRequest{Name: options.PVName}
	volumeConfig, err := p.ubiquityClient.GetVolumeConfig(getVolumeConfigRequest)
	if err != nil {
		return nil, fmt.Errorf("error getting volume config details: %v", err)
	}

	flexVolumeConfig := make(map[string]string)
	flexVolumeConfig["volumeName"] = options.PVName
	for key, value := range volumeConfig {
		flexVolumeConfig[key] = fmt.Sprintf("%v", value)
	}

	return flexVolumeConfig, nil
}
<commit_msg> #83: change PV name to match PVC name<commit_after>/**
 * Copyright 2017 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package volume

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"

	k8sresources "github.com/IBM/ubiquity-k8s/resources"
	"github.com/IBM/ubiquity/resources"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/client-go/pkg/api/v1"
)

const (

	// are we allowed to set this? else make up our own
	annCreatedBy = "kubernetes.io/createdby"
	createdBy    = k8sresources.UbiquityProvisionerName

	// Name of the file where an nfsProvisioner will store its identity
	identityFile = "k8sresources.UbiquityProvisionerName" + ".identity"

	// VolumeGidAnnotationKey is the key of the annotation on the PersistentVolume
	// object that specifies a supplemental GID.
	VolumeGidAnnotationKey = "pv.beta.kubernetes.io/gid"

	// A PV annotation for the identity of the flexProvisioner that provisioned it
	annProvisionerId = "Provisioner_Id"

	podIPEnv     = "POD_IP"
	serviceEnv   = "SERVICE_NAME"
	namespaceEnv = "POD_NAMESPACE"
	nodeEnv      = "NODE_NAME"
)

func NewFlexProvisioner(logger *log.Logger, ubiquityClient resources.StorageClient, config resources.UbiquityPluginConfig) (controller.Provisioner, error) {
	return newFlexProvisionerInternal(logger, ubiquityClient, config)
}

func newFlexProvisionerInternal(logger *log.Logger, ubiquityClient resources.StorageClient, config resources.UbiquityPluginConfig) (*flexProvisioner, error) {
	var identity types.UID
	identityPath := path.Join(config.LogPath, identityFile)
	if _, err := os.Stat(identityPath); os.IsNotExist(err) {
		identity = uuid.NewUUID()
		err := ioutil.WriteFile(identityPath, []byte(identity), 0600)
		if err != nil {
			logger.Printf("Error writing identity file %s! %v", identityPath, err)
		}
	} else {
		read, err := ioutil.ReadFile(identityPath)
		if err != nil {
			logger.Printf("Error reading identity file %s! %v", config.LogPath, err)
		}
		identity = types.UID(strings.TrimSpace(string(read)))
	}
	provisioner := &flexProvisioner{
		logger:         logger,
		identity:       identity,
		ubiquityClient: ubiquityClient,
		ubiquityConfig: config,
		podIPEnv:       podIPEnv,
		serviceEnv:     serviceEnv,
		namespaceEnv:   namespaceEnv,
		nodeEnv:        nodeEnv,
	}

	activateRequest := resources.ActivateRequest{Backends: config.Backends}
	err := provisioner.ubiquityClient.Activate(activateRequest)

	return provisioner, err
}

type flexProvisioner struct {
	logger   *log.Logger
	identity types.UID
	// Whether the provisioner is running out of cluster and so cannot rely on
	// the existence of any of the pod, service, namespace, node env variables.
	outOfCluster bool

	ubiquityClient resources.StorageClient
	ubiquityConfig resources.UbiquityPluginConfig

	// Environment variables the provisioner pod needs valid values for in order to
	// put a service cluster IP as the server of provisioned NFS PVs, passed in
	// via downward API. If serviceEnv is set, namespaceEnv must be too.
	podIPEnv     string
	serviceEnv   string
	namespaceEnv string
	nodeEnv      string
}

// Provision creates a volume i.e. the storage asset and returns a PV object for
// the volume.
func (p *flexProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) {
	if options.PVC == nil {
		return nil, fmt.Errorf("options missing PVC %#v", options)
	}

	// change volume name to match the PVC name
	options.PVName = options.PVC.Name

	capacity, exists := options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	if !exists {
		return nil, fmt.Errorf("options.PVC.Spec.Resources.Requests does not contain capacity")
	}
	fmt.Printf("PVC with capacity %d", capacity.Value())
	capacityMB := capacity.Value() / (1024 * 1024)

	volume_details, err := p.createVolume(options, capacityMB)
	if err != nil {
		return nil, err
	}

	annotations := make(map[string]string)
	annotations[annCreatedBy] = createdBy
	annotations[annProvisionerId] = k8sresources.UbiquityProvisionerName

	pv := &v1.PersistentVolume{
		ObjectMeta: v1.ObjectMeta{
			Name:        options.PVName,
			Labels:      map[string]string{},
			Annotations: annotations,
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
			AccessModes:                   options.PVC.Spec.AccessModes,
			Capacity: v1.ResourceList{
				v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)],
			},
			PersistentVolumeSource: v1.PersistentVolumeSource{
				FlexVolume: &v1.FlexVolumeSource{
					Driver:    k8sresources.UbiquityK8sFlexVolumeDriverFullName,
					FSType:    "",
					SecretRef: nil,
					ReadOnly:  false,
					Options:   volume_details,
				},
			},
		},
	}

	return pv, nil
}

// Delete removes the directory that was created by Provision backing the given
// PV.
func (p *flexProvisioner) Delete(volume *v1.PersistentVolume) error {
	if volume.Name == "" {
		return fmt.Errorf("volume name cannot be empty %#v", volume)
	}

	if volume.Spec.PersistentVolumeReclaimPolicy != v1.PersistentVolumeReclaimRetain {
		getVolumeRequest := resources.GetVolumeRequest{Name: volume.Name}
		volume, err := p.ubiquityClient.GetVolume(getVolumeRequest)
		if err != nil {
			fmt.Printf("error-retrieving-volume-info")
			return err
		}
		removeVolumeRequest := resources.RemoveVolumeRequest{Name: volume.Name}
		err = p.ubiquityClient.RemoveVolume(removeVolumeRequest)
		if err != nil {
			return err
		}
		return nil

	}

	return nil
}

func (p *flexProvisioner) createVolume(options controller.VolumeOptions, capacity int64) (map[string]string, error) {
	ubiquityParams := make(map[string]interface{})
	if capacity != 0 {
		ubiquityParams["quota"] = fmt.Sprintf("%dM", capacity)    // SSc backend expect quota option
		ubiquityParams["size"] = fmt.Sprintf("%d", capacity/1024) // SCBE backend expect size option
	}
	for key, value := range options.Parameters {
		ubiquityParams[key] = value
	}
	backendName, exists := ubiquityParams["backend"]
	if !exists {
		return nil, fmt.Errorf("backend is not specified")
	}
	b := backendName.(string)
	createVolumeRequest := resources.CreateVolumeRequest{Name: options.PVName, Backend: b, Opts: ubiquityParams}
	err := p.ubiquityClient.CreateVolume(createVolumeRequest)
	if err != nil {
		return nil, fmt.Errorf("error creating volume: %v", err)
	}

	getVolumeConfigRequest := resources.GetVolumeConfigRequest{Name: options.PVName}
	volumeConfig, err := p.ubiquityClient.GetVolumeConfig(getVolumeConfigRequest)
	if err != nil {
		return nil, fmt.Errorf("error getting volume config details: %v", err)
	}

	flexVolumeConfig := make(map[string]string)
	flexVolumeConfig["volumeName"] = options.PVName
	for key, value := range volumeConfig {
		flexVolumeConfig[key] = fmt.Sprintf("%v", value)
	}

	return flexVolumeConfig, nil
}
<|endoftext|>