<commit_before>package shell_test

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"testing"
	"time"

	"github.com/buildkite/agent/bootstrap/shell"
	"github.com/buildkite/bintest"
)

func TestRunAndCaptureWithTTY(t *testing.T) {
	sshKeygen, err := bintest.CompileProxy("ssh-keygen")
	if err != nil {
		t.Fatal(err)
	}
	defer sshKeygen.Close()

	sh := newShellForTest(t)
	sh.PTY = true

	go func() {
		call := <-sshKeygen.Ch
		fmt.Fprintln(call.Stdout, "Llama party! 🎉")
		call.Exit(0)
	}()

	actual, err := sh.RunAndCapture(sshKeygen.Path, "-f", "my_hosts", "-F", "llamas.com")
	if err != nil {
		t.Error(err)
	}

	if expected := "Llama party! 🎉"; string(actual) != expected {
		t.Errorf("Expected %q, got %q", expected, actual)
	}
}

func TestRunAndCaptureWithExitCode(t *testing.T) {
	sshKeygen, err := bintest.CompileProxy("ssh-keygen")
	if err != nil {
		t.Fatal(err)
	}
	defer sshKeygen.Close()

	sh := newShellForTest(t)

	go func() {
		call := <-sshKeygen.Ch
		fmt.Fprintln(call.Stdout, "Llama drama! 🚨")
		call.Exit(24)
	}()

	_, err = sh.RunAndCapture(sshKeygen.Path)
	if err == nil {
		t.Error("Expected an error, got nil")
	}

	if exitCode := shell.GetExitCode(err); exitCode != 24 {
		t.Fatalf("Expected %d, got %d", 24, exitCode)
	}
}

func TestRun(t *testing.T) {
	sshKeygen, err := bintest.CompileProxy("ssh-keygen")
	if err != nil {
		t.Fatal(err)
	}
	defer sshKeygen.Close()

	out := &bytes.Buffer{}

	sh := newShellForTest(t)
	sh.PTY = false
	sh.Writer = out
	sh.Logger = &shell.WriterLogger{Writer: out, Ansi: false}

	go func() {
		call := <-sshKeygen.Ch
		fmt.Fprintln(call.Stdout, "Llama party! 🎉")
		call.Exit(0)
	}()

	if err = sh.Run(sshKeygen.Path, "-f", "my_hosts", "-F", "llamas.com"); err != nil {
		t.Fatal(err)
	}

	actual := out.String()

	promptPrefix := "$"
	if runtime.GOOS == "windows" {
		promptPrefix = ">"
	}

	if expected := promptPrefix + " " + sshKeygen.Path + " -f my_hosts -F llamas.com\nLlama party! 🎉\n"; actual != expected {
		t.Fatalf("Expected %q, got %q", expected, actual)
	}
}

func TestContextCancelTerminates(t *testing.T) {
	sleepCmd, err := bintest.CompileProxy("sleep")
	if err != nil {
		t.Fatal(err)
	}
	defer sleepCmd.Close()

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	sh, err := shell.NewWithContext(ctx)
	if err != nil {
		t.Fatal(err)
	}

	sh.Logger = shell.DiscardLogger

	go func() {
		call := <-sleepCmd.Ch
		time.Sleep(time.Second * 60)
		call.Exit(0)
	}()

	cancel()

	err = sh.Run(sleepCmd.Path)
	if !shell.IsExitSignaled(err) {
		t.Fatalf("Expected signal exit, got %#v", err)
	}
}

func TestInterrupt(t *testing.T) {
	sleepCmd, err := bintest.CompileProxy("sleep")
	if err != nil {
		t.Fatal(err)
	}
	defer sleepCmd.Close()

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	sh, err := shell.NewWithContext(ctx)
	if err != nil {
		t.Fatal(err)
	}

	sh.Logger = shell.DiscardLogger

	go func() {
		call := <-sleepCmd.Ch
		time.Sleep(time.Second * 10)
		call.Exit(0)
	}()

	// kill the process after 2 seconds
	go func() {
		<-time.After(time.Second * 2)
		t.Error("Ran too long, should have terminated after 50ms")
		cancel()
	}()

	// interrupt the process after 50ms
	go func() {
		<-time.After(time.Millisecond * 50)
		sh.Interrupt()
	}()

	if err = sh.Run(sleepCmd.Path); err != nil {
		t.Fatal(err)
	}
}

func TestDefaultWorkingDirFromSystem(t *testing.T) {
	sh, err := shell.New()
	if err != nil {
		t.Fatal(err)
	}

	currentWd, _ := os.Getwd()
	if actual := sh.Getwd(); actual != currentWd {
		t.Fatalf("Expected working dir %q, got %q", currentWd, actual)
	}
}

func TestWorkingDir(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "shelltest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tempDir)

	// macos has a symlinked temp dir
	if runtime.GOOS == "darwin" {
		tempDir, _ = filepath.EvalSymlinks(tempDir)
	}

	dirs := []string{tempDir, "my", "test", "dirs"}

	if err := os.MkdirAll(filepath.Join(dirs...), 0700); err != nil {
		t.Fatal(err)
	}

	currentWd, _ := os.Getwd()

	sh, err := shell.New()
	sh.Logger = shell.DiscardLogger

	if err != nil {
		t.Fatal(err)
	}

	for idx := range dirs {
		dir := filepath.Join(dirs[0 : idx+1]...)

		if err := sh.Chdir(dir); err != nil {
			t.Fatal(err)
		}

		if actual := sh.Getwd(); actual != dir {
			t.Fatalf("Expected working dir %q, got %q", dir, actual)
		}

		var out string

		// there is no pwd for windows, and getting it requires using a shell builtin
		if runtime.GOOS == "windows" {
			out, err = sh.RunAndCapture("cmd", "/c", "echo", "%cd%")
			if err != nil {
				t.Fatal(err)
			}
		} else {
			out, err = sh.RunAndCapture("pwd")
			if err != nil {
				t.Fatal(err)
			}
		}

		if actual := out; actual != dir {
			t.Fatalf("Expected working dir (from pwd command) %q, got %q", dir, actual)
		}
	}

	afterWd, _ := os.Getwd()
	if afterWd != currentWd {
		t.Fatalf("Expected working dir to be the same as before shell commands ran")
	}
}

func TestLockFileRetriesAndTimesOut(t *testing.T) {
	dir, err := ioutil.TempDir("", "shelltest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	sh := newShellForTest(t)
	sh.Logger = shell.DiscardLogger

	lockPath := filepath.Join(dir, "my.lock")

	// acquire a lock in another process
	cmd, err := acquireLockInOtherProcess(lockPath)
	if err != nil {
		t.Fatal(err)
	}

	defer cmd.Process.Kill()

	// acquire lock
	_, err = sh.LockFile(lockPath, time.Second*2)
	if err != context.DeadlineExceeded {
		t.Fatalf("Expected DeadlineExceeded error, got %v", err)
	}
}

func acquireLockInOtherProcess(lockfile string) (*exec.Cmd, error) {
	cmd := exec.Command(os.Args[0], "-test.run=TestAcquiringLockHelperProcess", "--", lockfile)
	cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}

	err := cmd.Start()
	if err != nil {
		return cmd, err
	}

	// wait for the above process to get a lock
	for {
		if _, err = os.Stat(lockfile); os.IsNotExist(err) {
			time.Sleep(time.Millisecond * 10)
			continue
		}
		break
	}

	return cmd, nil
}

// TestAcquiringLockHelperProcess isn't a real test. It's used as a helper process
func TestAcquiringLockHelperProcess(t *testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}

	fileName := os.Args[len(os.Args)-1]
	sh := newShellForTest(t)

	log.Printf("Locking %s", fileName)
	if _, err := sh.LockFile(fileName, time.Second*10); err != nil {
		os.Exit(1)
	}

	log.Printf("Acquired lock %s", fileName)
	c := make(chan struct{})
	<-c
}

func newShellForTest(t *testing.T) *shell.Shell {
	sh, err := shell.New()
	if err != nil {
		t.Fatal(err)
	}
	sh.Logger = shell.DiscardLogger
	return sh
}
<commit_msg>Fix incorrect test<commit_after>package shell_test

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"testing"
	"time"

	"github.com/buildkite/agent/bootstrap/shell"
	"github.com/buildkite/bintest"
)

func TestRunAndCaptureWithTTY(t *testing.T) {
	sshKeygen, err := bintest.CompileProxy("ssh-keygen")
	if err != nil {
		t.Fatal(err)
	}
	defer sshKeygen.Close()

	sh := newShellForTest(t)
	sh.PTY = true

	go func() {
		call := <-sshKeygen.Ch
		fmt.Fprintln(call.Stdout, "Llama party! 🎉")
		call.Exit(0)
	}()

	actual, err := sh.RunAndCapture(sshKeygen.Path, "-f", "my_hosts", "-F", "llamas.com")
	if err != nil {
		t.Error(err)
	}

	if expected := "Llama party! 🎉"; string(actual) != expected {
		t.Errorf("Expected %q, got %q", expected, actual)
	}
}

func TestRunAndCaptureWithExitCode(t *testing.T) {
	sshKeygen, err := bintest.CompileProxy("ssh-keygen")
	if err != nil {
		t.Fatal(err)
	}
	defer sshKeygen.Close()

	sh := newShellForTest(t)

	go func() {
		call := <-sshKeygen.Ch
		fmt.Fprintln(call.Stdout, "Llama drama! 🚨")
		call.Exit(24)
	}()

	_, err = sh.RunAndCapture(sshKeygen.Path)
	if err == nil {
		t.Error("Expected an error, got nil")
	}

	if exitCode := shell.GetExitCode(err); exitCode != 24 {
		t.Fatalf("Expected %d, got %d", 24, exitCode)
	}
}

func TestRun(t *testing.T) {
	sshKeygen, err := bintest.CompileProxy("ssh-keygen")
	if err != nil {
		t.Fatal(err)
	}
	defer sshKeygen.Close()

	out := &bytes.Buffer{}

	sh := newShellForTest(t)
	sh.PTY = false
	sh.Writer = out
	sh.Logger = &shell.WriterLogger{Writer: out, Ansi: false}

	go func() {
		call := <-sshKeygen.Ch
		fmt.Fprintln(call.Stdout, "Llama party! 🎉")
		call.Exit(0)
	}()

	if err = sh.Run(sshKeygen.Path, "-f", "my_hosts", "-F", "llamas.com"); err != nil {
		t.Fatal(err)
	}

	actual := out.String()

	promptPrefix := "$"
	if runtime.GOOS == "windows" {
		promptPrefix = ">"
	}

	if expected := promptPrefix + " " + sshKeygen.Path + " -f my_hosts -F llamas.com\nLlama party! 🎉\n"; actual != expected {
		t.Fatalf("Expected %q, got %q", expected, actual)
	}
}

func TestContextCancelTerminates(t *testing.T) {
	sleepCmd, err := bintest.CompileProxy("sleep")
	if err != nil {
		t.Fatal(err)
	}
	defer sleepCmd.Close()

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	sh, err := shell.NewWithContext(ctx)
	if err != nil {
		t.Fatal(err)
	}

	sh.Logger = shell.DiscardLogger

	go func() {
		call := <-sleepCmd.Ch
		time.Sleep(time.Second * 60)
		call.Exit(0)
	}()

	cancel()

	err = sh.Run(sleepCmd.Path)
	if !shell.IsExitSignaled(err) {
		t.Fatalf("Expected signal exit, got %#v", err)
	}
}

func TestInterrupt(t *testing.T) {
	sleepCmd, err := bintest.CompileProxy("sleep")
	if err != nil {
		t.Fatal(err)
	}
	defer sleepCmd.Close()

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	sh, err := shell.NewWithContext(ctx)
	if err != nil {
		t.Fatal(err)
	}

	sh.Logger = shell.DiscardLogger

	go func() {
		call := <-sleepCmd.Ch
		time.Sleep(time.Second * 10)
		call.Exit(0)
	}()

	// kill the process after 2 seconds
	go func() {
		<-time.After(time.Second * 2)
		t.Error("Ran too long, should have terminated after 50ms")
		cancel()
	}()

	// interrupt the process after 50ms
	go func() {
		<-time.After(time.Millisecond * 50)
		sh.Interrupt()
	}()

	err = sh.Run(sleepCmd.Path)
	if err == nil {
		t.Error("Expected an error")
	}
}

func TestDefaultWorkingDirFromSystem(t *testing.T) {
	sh, err := shell.New()
	if err != nil {
		t.Fatal(err)
	}

	currentWd, _ := os.Getwd()
	if actual := sh.Getwd(); actual != currentWd {
		t.Fatalf("Expected working dir %q, got %q", currentWd, actual)
	}
}

func TestWorkingDir(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "shelltest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tempDir)

	// macos has a symlinked temp dir
	if runtime.GOOS == "darwin" {
		tempDir, _ = filepath.EvalSymlinks(tempDir)
	}

	dirs := []string{tempDir, "my", "test", "dirs"}

	if err := os.MkdirAll(filepath.Join(dirs...), 0700); err != nil {
		t.Fatal(err)
	}

	currentWd, _ := os.Getwd()

	sh, err := shell.New()
	sh.Logger = shell.DiscardLogger

	if err != nil {
		t.Fatal(err)
	}

	for idx := range dirs {
		dir := filepath.Join(dirs[0 : idx+1]...)

		if err := sh.Chdir(dir); err != nil {
			t.Fatal(err)
		}

		if actual := sh.Getwd(); actual != dir {
			t.Fatalf("Expected working dir %q, got %q", dir, actual)
		}

		var out string

		// there is no pwd for windows, and getting it requires using a shell builtin
		if runtime.GOOS == "windows" {
			out, err = sh.RunAndCapture("cmd", "/c", "echo", "%cd%")
			if err != nil {
				t.Fatal(err)
			}
		} else {
			out, err = sh.RunAndCapture("pwd")
			if err != nil {
				t.Fatal(err)
			}
		}

		if actual := out; actual != dir {
			t.Fatalf("Expected working dir (from pwd command) %q, got %q", dir, actual)
		}
	}

	afterWd, _ := os.Getwd()
	if afterWd != currentWd {
		t.Fatalf("Expected working dir to be the same as before shell commands ran")
	}
}

func TestLockFileRetriesAndTimesOut(t *testing.T) {
	dir, err := ioutil.TempDir("", "shelltest")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)

	sh := newShellForTest(t)
	sh.Logger = shell.DiscardLogger

	lockPath := filepath.Join(dir, "my.lock")

	// acquire a lock in another process
	cmd, err := acquireLockInOtherProcess(lockPath)
	if err != nil {
		t.Fatal(err)
	}

	defer cmd.Process.Kill()

	// acquire lock
	_, err = sh.LockFile(lockPath, time.Second*2)
	if err != context.DeadlineExceeded {
		t.Fatalf("Expected DeadlineExceeded error, got %v", err)
	}
}

func acquireLockInOtherProcess(lockfile string) (*exec.Cmd, error) {
	cmd := exec.Command(os.Args[0], "-test.run=TestAcquiringLockHelperProcess", "--", lockfile)
	cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}

	err := cmd.Start()
	if err != nil {
		return cmd, err
	}

	// wait for the above process to get a lock
	for {
		if _, err = os.Stat(lockfile); os.IsNotExist(err) {
			time.Sleep(time.Millisecond * 10)
			continue
		}
		break
	}

	return cmd, nil
}

// TestAcquiringLockHelperProcess isn't a real test. It's used as a helper process
func TestAcquiringLockHelperProcess(t *testing.T) {
	if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
		return
	}

	fileName := os.Args[len(os.Args)-1]
	sh := newShellForTest(t)

	log.Printf("Locking %s", fileName)
	if _, err := sh.LockFile(fileName, time.Second*10); err != nil {
		os.Exit(1)
	}

	log.Printf("Acquired lock %s", fileName)
	c := make(chan struct{})
	<-c
}

func newShellForTest(t *testing.T) *shell.Shell {
	sh, err := shell.New()
	if err != nil {
		t.Fatal(err)
	}
	sh.Logger = shell.DiscardLogger
	return sh
}
<|endoftext|>


<commit_before>// Package logentries provides the log driver for forwarding server logs
// to logentries endpoints.
package logentries

import (
	"fmt"
	"strconv"

	"github.com/bsphere/le_go"
	"github.com/docker/docker/daemon/logger"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type logentries struct {
	tag           string
	containerID   string
	containerName string
	writer        *le_go.Logger
	extra         map[string]string
	lineOnly      bool
}

const (
	name     = "logentries"
	token    = "logentries-token"
	lineonly = "line-only"
)

func init() {
	if err := logger.RegisterLogDriver(name, New); err != nil {
		logrus.Fatal(err)
	}
	if err := logger.RegisterLogOptValidator(name, ValidateLogOpt); err != nil {
		logrus.Fatal(err)
	}
}

// New creates a logentries logger using the configuration passed in on
// the context. The supported context configuration variable is
// logentries-token.
func New(info logger.Info) (logger.Logger, error) {
	logrus.WithField("container", info.ContainerID).
		WithField("token", info.Config[token]).
		WithField("line-only", info.Config[lineonly]).
		Debug("logging driver logentries configured")

	log, err := le_go.Connect(info.Config[token])
	if err != nil {
		return nil, errors.Wrap(err, "error connecting to logentries")
	}
	var lineOnly bool
	if lineOnly, err = strconv.ParseBool(info.Config[lineonly]); err != nil {
		return nil, errors.Wrap(err, "error parsing lineonly option")
	}
	return &logentries{
		containerID:   info.ContainerID,
		containerName: info.ContainerName,
		writer:        log,
		lineOnly:      lineOnly,
	}, nil
}

func (f *logentries) Log(msg *logger.Message) error {
	if !f.lineOnly {
		data := map[string]string{
			"container_id":   f.containerID,
			"container_name": f.containerName,
			"source":         msg.Source,
			"log":            string(msg.Line),
		}
		for k, v := range f.extra {
			data[k] = v
		}
		ts := msg.Timestamp
		logger.PutMessage(msg)
		f.writer.Println(f.tag, ts, data)
	} else {
		line := string(msg.Line)
		logger.PutMessage(msg)
		f.writer.Println(line)
	}
	return nil
}

func (f *logentries) Close() error {
	return f.writer.Close()
}

func (f *logentries) Name() string {
	return name
}

// ValidateLogOpt looks for logentries specific log option logentries-address.
func ValidateLogOpt(cfg map[string]string) error {
	for key := range cfg {
		switch key {
		case "env":
		case "env-regex":
		case "labels":
		case "tag":
		case key:
		default:
			return fmt.Errorf("unknown log opt '%s' for logentries log driver", key)
		}
	}

	if cfg[token] == "" {
		return fmt.Errorf("Missing logentries token")
	}

	return nil
}
<commit_msg>Logentries line-only logopt fix to maintain backwards compatibility Changed logic to ignore empty value<commit_after>// Package logentries provides the log driver for forwarding server logs
// to logentries endpoints.
package logentries

import (
	"fmt"
	"strconv"

	"github.com/bsphere/le_go"
	"github.com/docker/docker/daemon/logger"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type logentries struct {
	tag           string
	containerID   string
	containerName string
	writer        *le_go.Logger
	extra         map[string]string
	lineOnly      bool
}

const (
	name     = "logentries"
	token    = "logentries-token"
	lineonly = "line-only"
)

func init() {
	if err := logger.RegisterLogDriver(name, New); err != nil {
		logrus.Fatal(err)
	}
	if err := logger.RegisterLogOptValidator(name, ValidateLogOpt); err != nil {
		logrus.Fatal(err)
	}
}

// New creates a logentries logger using the configuration passed in on
// the context. The supported context configuration variable is
// logentries-token.
func New(info logger.Info) (logger.Logger, error) {
	logrus.WithField("container", info.ContainerID).
		WithField("token", info.Config[token]).
		WithField("line-only", info.Config[lineonly]).
		Debug("logging driver logentries configured")

	log, err := le_go.Connect(info.Config[token])
	if err != nil {
		return nil, errors.Wrap(err, "error connecting to logentries")
	}
	var lineOnly bool
	if info.Config[lineonly] != "" {
		if lineOnly, err = strconv.ParseBool(info.Config[lineonly]); err != nil {
			return nil, errors.Wrap(err, "error parsing lineonly option")
		}
	}
	return &logentries{
		containerID:   info.ContainerID,
		containerName: info.ContainerName,
		writer:        log,
		lineOnly:      lineOnly,
	}, nil
}

func (f *logentries) Log(msg *logger.Message) error {
	if !f.lineOnly {
		data := map[string]string{
			"container_id":   f.containerID,
			"container_name": f.containerName,
			"source":         msg.Source,
			"log":            string(msg.Line),
		}
		for k, v := range f.extra {
			data[k] = v
		}
		ts := msg.Timestamp
		logger.PutMessage(msg)
		f.writer.Println(f.tag, ts, data)
	} else {
		line := string(msg.Line)
		logger.PutMessage(msg)
		f.writer.Println(line)
	}
	return nil
}

func (f *logentries) Close() error {
	return f.writer.Close()
}

func (f *logentries) Name() string {
	return name
}

// ValidateLogOpt looks for logentries specific log option logentries-address.
func ValidateLogOpt(cfg map[string]string) error {
	for key := range cfg {
		switch key {
		case "env":
		case "env-regex":
		case "labels":
		case "tag":
		case key:
		default:
			return fmt.Errorf("unknown log opt '%s' for logentries log driver", key)
		}
	}

	if cfg[token] == "" {
		return fmt.Errorf("Missing logentries token")
	}

	return nil
}
<|endoftext|>


<commit_before>package mm

import (
	"fmt"
	"github.com/freeformz/shh/config"
	"strings"
	"time"
)

type Measurement struct {
	When   time.Time
	Poller string
	What   []string
	Value  interface{}
}

func (m *Measurement) String() string {
	msg := fmt.Sprintf("when=%s measure=%s", m.Timestamp(), m.Measured())
	switch m.Value.(type) {
	case float64:
		msg = fmt.Sprintf("%s val=%f", msg, m.Value.(float64))
	case uint64:
		msg = fmt.Sprintf("%s val=%d", msg, m.Value.(uint64))
	}
	if config.Source != "" {
		return fmt.Sprintf("%s source=%s", msg, config.Source)
	}
	return msg
}

func (m *Measurement) Measured() string {
	v := fmt.Sprintf("%s.%s", m.Poller, strings.Join(m.What, "."))
	if config.Prefix != "" {
		v = fmt.Sprintf("%s.%s", config.Prefix, v)
	}
	return v
}

func (m *Measurement) Source() string {
	return config.Source
}

func (current *Measurement) Difference(last *Measurement) uint64 {
	// This is a crappy way to handle wraps and resets when we don't know
	// what the max value is (32, 64 or 128 bit)
	// Leads to a little, loss, but should be minimal overall
	cv := current.Value.(uint64)
	lv := last.Value.(uint64)
	if cv < lv {
		return cv
	}
	return cv - lv
}

func (m *Measurement) Timestamp() string {
	return m.When.Format(time.RFC3339)
}
<commit_msg>Move this off to a function and return UnixNano time<commit_after>package mm

import (
	"fmt"
	"github.com/freeformz/shh/config"
	"strings"
	"time"
)

type Measurement struct {
	When   time.Time
	Poller string
	What   []string
	Value  interface{}
}

func (m *Measurement) String() string {
	msg := fmt.Sprintf("when=%s measure=%s val=%s", m.Timestamp(), m.Measured(), m.SValue())
	if config.Source != "" {
		return fmt.Sprintf("%s source=%s", msg, config.Source)
	}
	return msg
}

func (m *Measurement) SValue() string {
	switch m.Value.(type) {
	case float64:
		return fmt.Sprintf("%f", m.Value.(float64))
	case uint64:
		return fmt.Sprintf("%d", m.Value.(uint64))
	}
	return ""
}

func (m *Measurement) Measured() string {
	v := fmt.Sprintf("%s.%s", m.Poller, strings.Join(m.What, "."))
	if config.Prefix != "" {
		v = fmt.Sprintf("%s.%s", config.Prefix, v)
	}
	return v
}

func (m *Measurement) Source() string {
	return config.Source
}

func (current *Measurement) Difference(last *Measurement) uint64 {
	// This is a crappy way to handle wraps and resets when we don't know
	// what the max value is (32, 64 or 128 bit)
	// Leads to a little, loss, but should be minimal overall
	cv := current.Value.(uint64)
	lv := last.Value.(uint64)
	if cv < lv {
		return cv
	}
	return cv - lv
}

func (m *Measurement) Timestamp() string {
	return m.When.Format(time.RFC3339)
}

func (m *Measurement) UnixNano() int64 {
	return m.When.UnixNano()
}
<|endoftext|>


<commit_before>package objectstorage

import (
	"bytes"
	"encoding/base64"
	"encoding/hex"
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
)

// listMultipartUploadsRequest wrapper creates a new listMultipartUploads request
func (a *lowLevelAPI) listMultipartUploadsRequest(bucket, keymarker, uploadIDMarker, prefix, delimiter string, maxuploads int) (*request, error) {
	// resourceQuery - get resources properly escaped and lined up before using them in http request
	resourceQuery := func() string {
		switch {
		case keymarker != "":
			keymarker = fmt.Sprintf("&key-marker=%s", url.QueryEscape(keymarker))
			fallthrough
		case uploadIDMarker != "":
			uploadIDMarker = fmt.Sprintf("&upload-id-marker=%s", url.QueryEscape(uploadIDMarker))
			fallthrough
		case prefix != "":
			prefix = fmt.Sprintf("&prefix=%s", url.QueryEscape(prefix))
			fallthrough
		case delimiter != "":
			delimiter = fmt.Sprintf("&delimiter=%s", url.QueryEscape(delimiter))
		}
		return fmt.Sprintf("?uploads&max-uploads=%d", maxuploads) + keymarker + uploadIDMarker + prefix + delimiter
	}
	op := &operation{
		HTTPServer: a.config.MustGetEndpoint(),
		HTTPMethod: "GET",
		HTTPPath:   "/" + bucket + resourceQuery(),
	}
	r, err := newRequest(op, a.config, nil)
	if err != nil {
		return nil, err
	}
	return r, nil
}

// listMultipartUploads - (List Multipart Uploads) - Lists some or all (up to 1000) in-progress multipart uploads in a bucket.
//
// You can use the request parameters as selection criteria to return a subset of the uploads in a bucket.
// request paramters :-
// ---------
// ?key-marker - Specifies the multipart upload after which listing should begin
// ?upload-id-marker - Together with key-marker specifies the multipart upload after which listing should begin
// ?delimiter - A delimiter is a character you use to group keys.
// ?prefix - Limits the response to keys that begin with the specified prefix.
// ?max-uploads - Sets the maximum number of multipart uploads returned in the response body.
func (a *lowLevelAPI) listMultipartUploads(bucket, keymarker, uploadIDMarker, prefix, delimiter string, maxuploads int) (*listMultipartUploadsResult, error) {
	req, err := a.listMultipartUploadsRequest(bucket, keymarker, uploadIDMarker, prefix, delimiter, maxuploads)
	if err != nil {
		return nil, err
	}
	resp, err := req.Do()
	if err != nil {
		return nil, err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return nil, responseToError(resp)
		}
	}
	listMultipartUploadsResult := new(listMultipartUploadsResult)
	decoder := xml.NewDecoder(resp.Body)
	err = decoder.Decode(listMultipartUploadsResult)
	if err != nil {
		return nil, err
	}

	// close body while returning, along with any error
	return listMultipartUploadsResult, resp.Body.Close()
}

// initiateMultipartRequest wrapper creates a new initiateMultiPart request
func (a *lowLevelAPI) initiateMultipartRequest(bucket, object string) (*request, error) {
	op := &operation{
		HTTPServer: a.config.MustGetEndpoint(),
		HTTPMethod: "POST",
		HTTPPath:   "/" + bucket + "/" + object + "?uploads",
	}
	return newRequest(op, a.config, nil)
}

// initiateMultipartUpload initiates a multipart upload and returns an upload ID
func (a *lowLevelAPI) initiateMultipartUpload(bucket, object string) (*initiateMultipartUploadResult, error) {
	req, err := a.initiateMultipartRequest(bucket, object)
	if err != nil {
		return nil, err
	}
	resp, err := req.Do()
	if err != nil {
		return nil, err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return nil, responseToError(resp)
		}
	}
	initiateMultipartUploadResult := new(initiateMultipartUploadResult)
	decoder := xml.NewDecoder(resp.Body)
	err = decoder.Decode(initiateMultipartUploadResult)
	if err != nil {
		return nil, err
	}
	return initiateMultipartUploadResult, resp.Body.Close()
}

// completeMultipartUploadRequest wrapper creates a new CompleteMultipartUpload request
func (a *lowLevelAPI) completeMultipartUploadRequest(bucket, object, uploadID string, complete *completeMultipartUpload) (*request, error) {
	op := &operation{
		HTTPServer: a.config.MustGetEndpoint(),
		HTTPMethod: "POST",
		HTTPPath:   "/" + bucket + "/" + object + "?uploadId=" + uploadID,
	}
	completeMultipartUploadBytes, err := xml.Marshal(complete)
	if err != nil {
		return nil, err
	}
	completeMultipartUploadBuffer := bytes.NewReader(completeMultipartUploadBytes)
	r, err := newRequest(op, a.config, completeMultipartUploadBuffer)
	if err != nil {
		return nil, err
	}
	r.req.ContentLength = int64(completeMultipartUploadBuffer.Len())
	return r, nil
}

// completeMultipartUpload completes a multipart upload by assembling previously uploaded parts.
func (a *lowLevelAPI) completeMultipartUpload(bucket, object, uploadID string, c *completeMultipartUpload) (*completeMultipartUploadResult, error) {
	req, err := a.completeMultipartUploadRequest(bucket, object, uploadID, c)
	if err != nil {
		return nil, err
	}
	resp, err := req.Do()
	if err != nil {
		return nil, err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return nil, responseToError(resp)
		}
	}
	completeMultipartUploadResult := new(completeMultipartUploadResult)
	decoder := xml.NewDecoder(resp.Body)
	err = decoder.Decode(completeMultipartUploadResult)
	if err != nil {
		return nil, err
	}
	return completeMultipartUploadResult, resp.Body.Close()
}

// abortMultipartUploadRequest wrapper creates a new AbortMultipartUpload request
func (a *lowLevelAPI) abortMultipartUploadRequest(bucket, object, uploadID string) (*request, error) {
	op := &operation{
		HTTPServer: a.config.MustGetEndpoint(),
		HTTPMethod: "DELETE",
		HTTPPath:   "/" + bucket + "/" + object + "?uploadId=" + uploadID,
	}
	return newRequest(op, a.config, nil)
}

// abortMultipartUpload aborts a multipart upload for the given uploadID, all parts are deleted
func (a *lowLevelAPI) abortMultipartUpload(bucket, object, uploadID string) error {
	req, err := a.abortMultipartUploadRequest(bucket, object, uploadID)
	if err != nil {
		return err
	}
	resp, err := req.Do()
	if err != nil {
		return err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusNoContent {
			// Abort has no response body, handle it
			return fmt.Errorf("%s", resp.Status)
		}
	}
	return resp.Body.Close()
}

// listObjectPartsRequest wrapper creates a new ListObjectParts request
func (a *lowLevelAPI) listObjectPartsRequest(bucket, object, uploadID string) (*request, error) {
	op := &operation{
		HTTPServer: a.config.MustGetEndpoint(),
		HTTPMethod: "GET",
		HTTPPath:   "/" + bucket + "/" + object + "?uploadId=" + uploadID,
	}
	return newRequest(op, a.config, nil)
}

// listObjectParts lists the parts that have been uploaded for a specific multipart upload.
func (a *lowLevelAPI) listObjectParts(bucket, object, uploadID string) (*listObjectPartsResult, error) {
	req, err := a.listObjectPartsRequest(bucket, object, uploadID)
	if err != nil {
		return nil, err
	}
	resp, err := req.Do()
	if err != nil {
		return nil, err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return nil, responseToError(resp)
		}
	}
	listObjectPartsResult := new(listObjectPartsResult)
	decoder := xml.NewDecoder(resp.Body)
	err = decoder.Decode(listObjectPartsResult)
	if err != nil {
		return nil, err
	}
	return listObjectPartsResult, resp.Body.Close()
}

// uploadPartRequest wrapper creates a new UploadPart request
func (a *lowLevelAPI) uploadPartRequest(bucket, object, uploadID string, partNumber int, size int64, body io.ReadSeeker) (*request, error) {
	op := &operation{
		HTTPServer: a.config.MustGetEndpoint(),
		HTTPMethod: "PUT",
		HTTPPath:   "/" + bucket + "/" + object + "?partNumber=" + strconv.Itoa(partNumber) + "&uploadId=" + uploadID,
	}
	md5SumBytes, err := sumMD5Reader(body, size)
	if err != nil {
		return nil, err
	}
	r, err := newRequest(op, a.config, body)
	if err != nil {
		return nil, err
	}
	// set Content-MD5 as base64 encoded md5
	r.Set("Content-MD5", base64.StdEncoding.EncodeToString(md5SumBytes))
	r.req.ContentLength = size
	return r, nil
}

// uploadPart uploads a part in a multipart upload.
func (a *lowLevelAPI) uploadPart(bucket, object, uploadID string, partNumber int, size int64, body io.ReadSeeker) (*completePart, error) {
	req, err := a.uploadPartRequest(bucket, object, uploadID, partNumber, size, body)
	if err != nil {
		return nil, err
	}
	// get hex encoding for md5sum in base64
	md5SumBytes, err := base64.StdEncoding.DecodeString(req.Get("Content-MD5"))
	if err != nil {
		return nil, err
	}
	completePart := new(completePart)
	completePart.PartNumber = partNumber
	completePart.ETag = "\"" + hex.EncodeToString(md5SumBytes) + "\""

	// initiate the request
	resp, err := req.Do()
	if err != nil {
		return nil, err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return nil, responseToError(resp)
		}
	}
	return completePart, resp.Body.Close()
}
<commit_msg>Support partNumberMarker for listObjectParts query<commit_after>package objectstorage

import (
	"bytes"
	"encoding/base64"
	"encoding/hex"
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
)

// listMultipartUploadsRequest wrapper creates a new listMultipartUploads request
func (a *lowLevelAPI) listMultipartUploadsRequest(bucket, keymarker, uploadIDMarker, prefix, delimiter string, maxuploads int) (*request, error) {
	// resourceQuery - get resources properly escaped and lined up before using them in http request
	resourceQuery := func() string {
		switch {
		case keymarker != "":
			keymarker = fmt.Sprintf("&key-marker=%s", url.QueryEscape(keymarker))
			fallthrough
		case uploadIDMarker != "":
			uploadIDMarker = fmt.Sprintf("&upload-id-marker=%s", url.QueryEscape(uploadIDMarker))
			fallthrough
		case prefix != "":
			prefix = fmt.Sprintf("&prefix=%s", url.QueryEscape(prefix))
			fallthrough
		case delimiter != "":
			delimiter = fmt.Sprintf("&delimiter=%s", url.QueryEscape(delimiter))
		}
		return fmt.Sprintf("?uploads&max-uploads=%d", maxuploads) + keymarker + uploadIDMarker + prefix + delimiter
	}
	op := &operation{
		HTTPServer: a.config.MustGetEndpoint(),
		HTTPMethod: "GET",
		HTTPPath:   "/" + bucket + resourceQuery(),
	}
	r, err := newRequest(op, a.config, nil)
	if err != nil {
		return nil, err
	}
	return r, nil
}

// listMultipartUploads - (List Multipart Uploads) - Lists some or all (up to 1000) in-progress multipart uploads in a bucket.
//
// You can use the request parameters as selection criteria to return a subset of the uploads in a bucket.
// request paramters :-
// ---------
// ?key-marker - Specifies the multipart upload after which listing should begin
// ?upload-id-marker - Together with key-marker specifies the multipart upload after which listing should begin
// ?delimiter - A delimiter is a character you use to group keys.
// ?prefix - Limits the response to keys that begin with the specified prefix.
// ?max-uploads - Sets the maximum number of multipart uploads returned in the response body.
func (a *lowLevelAPI) listMultipartUploads(bucket, keymarker, uploadIDMarker, prefix, delimiter string, maxuploads int) (*listMultipartUploadsResult, error) {
	req, err := a.listMultipartUploadsRequest(bucket, keymarker, uploadIDMarker, prefix, delimiter, maxuploads)
	if err != nil {
		return nil, err
	}
	resp, err := req.Do()
	if err != nil {
		return nil, err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return nil, responseToError(resp)
		}
	}
	listMultipartUploadsResult := new(listMultipartUploadsResult)
	decoder := xml.NewDecoder(resp.Body)
	err = decoder.Decode(listMultipartUploadsResult)
	if err != nil {
		return nil, err
	}

	// close body while returning, along with any error
	return listMultipartUploadsResult, resp.Body.Close()
}

// initiateMultipartRequest wrapper creates a new initiateMultiPart request
func (a *lowLevelAPI) initiateMultipartRequest(bucket, object string) (*request, error) {
	op := &operation{
		HTTPServer: a.config.MustGetEndpoint(),
		HTTPMethod: "POST",
		HTTPPath:   "/" + bucket + "/" + object + "?uploads",
	}
	return newRequest(op, a.config, nil)
}

// initiateMultipartUpload initiates a multipart upload and returns an upload ID
func (a *lowLevelAPI) initiateMultipartUpload(bucket, object string) (*initiateMultipartUploadResult, error) {
	req, err := a.initiateMultipartRequest(bucket, object)
	if err != nil {
		return nil, err
	}
	resp, err := req.Do()
	if err != nil {
		return nil, err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return nil, responseToError(resp)
		}
	}
	initiateMultipartUploadResult := new(initiateMultipartUploadResult)
	decoder := xml.NewDecoder(resp.Body)
	err = decoder.Decode(initiateMultipartUploadResult)
	if err != nil {
		return nil, err
	}
	return initiateMultipartUploadResult, resp.Body.Close()
}

// completeMultipartUploadRequest wrapper creates a new CompleteMultipartUpload request
func (a *lowLevelAPI) completeMultipartUploadRequest(bucket, object, uploadID string, complete *completeMultipartUpload) (*request, error) {
	op := &operation{
		HTTPServer: a.config.MustGetEndpoint(),
		HTTPMethod: "POST",
		HTTPPath:   "/" + bucket + "/" + object + "?uploadId=" + uploadID,
	}
	completeMultipartUploadBytes, err := xml.Marshal(complete)
	if err != nil {
		return nil, err
	}
	completeMultipartUploadBuffer := bytes.NewReader(completeMultipartUploadBytes)
	r, err := newRequest(op, a.config, completeMultipartUploadBuffer)
	if err != nil {
		return nil, err
	}
	r.req.ContentLength = int64(completeMultipartUploadBuffer.Len())
	return r, nil
}

// completeMultipartUpload completes a multipart upload by assembling previously uploaded parts.
func (a *lowLevelAPI) completeMultipartUpload(bucket, object, uploadID string, c *completeMultipartUpload) (*completeMultipartUploadResult, error) {
	req, err := a.completeMultipartUploadRequest(bucket, object, uploadID, c)
	if err != nil {
		return nil, err
	}
	resp, err := req.Do()
	if err != nil {
		return nil, err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return nil, responseToError(resp)
		}
	}
	completeMultipartUploadResult := new(completeMultipartUploadResult)
	decoder := xml.NewDecoder(resp.Body)
	err = decoder.Decode(completeMultipartUploadResult)
	if err != nil {
		return nil, err
	}
	return completeMultipartUploadResult, resp.Body.Close()
}

// abortMultipartUploadRequest wrapper creates a new AbortMultipartUpload request
func (a *lowLevelAPI) abortMultipartUploadRequest(bucket, object, uploadID string) (*request, error) {
	op := &operation{
		HTTPServer: a.config.MustGetEndpoint(),
		HTTPMethod: "DELETE",
		HTTPPath:   "/" + bucket + "/" + object + "?uploadId=" + uploadID,
	}
	return newRequest(op, a.config, nil)
}

// abortMultipartUpload aborts a multipart upload for the given uploadID, all parts are deleted
func (a *lowLevelAPI) abortMultipartUpload(bucket, object, uploadID string) error {
	req, err := a.abortMultipartUploadRequest(bucket, object, uploadID)
	if err != nil {
		return err
	}
	resp, err := req.Do()
	if err != nil {
		return err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusNoContent {
			// Abort has no response body, handle it
			return fmt.Errorf("%s", resp.Status)
		}
	}
	return resp.Body.Close()
}

// listObjectPartsRequest wrapper creates a new ListObjectParts request
func (a *lowLevelAPI) listObjectPartsRequest(bucket, object, uploadID string, partNumberMarker, maxParts int) (*request, error) {
	// resourceQuery - get resources properly escaped and lined up before using them in http request
	resourceQuery := func() string {
		var partNumberMarkerStr string
		switch {
		case partNumberMarker != 0:
			partNumberMarkerStr = fmt.Sprintf("&part-number-marker=%d", partNumberMarker)
		}
		return fmt.Sprintf("?uploadsId=%s&max-parts=%d", uploadID, maxParts) + partNumberMarkerStr
	}
	op := &operation{
		HTTPServer: a.config.MustGetEndpoint(),
		HTTPMethod: "GET",
		HTTPPath:   "/" + bucket + "/" + object + resourceQuery(),
	}
	return newRequest(op, a.config, nil)
}

// listObjectParts (List Parts) - lists some or all (up to 1000) parts that have been uploaded for a specific multipart upload
//
// You can use the request parameters as selection criteria to return a subset of the uploads in a bucket.
// request paramters :-
// ---------
// ?part-number-marker - Specifies the part after which listing should begin.
func (a *lowLevelAPI) listObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (*listObjectPartsResult, error) {
	req, err := a.listObjectPartsRequest(bucket, object, uploadID, partNumberMarker, maxParts)
	if err != nil {
		return nil, err
	}
	resp, err := req.Do()
	if err != nil {
		return nil, err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return nil, responseToError(resp)
		}
	}
	listObjectPartsResult := new(listObjectPartsResult)
	decoder := xml.NewDecoder(resp.Body)
	err = decoder.Decode(listObjectPartsResult)
	if err != nil {
		return nil, err
	}
	return listObjectPartsResult, resp.Body.Close()
}

// uploadPartRequest wrapper creates a new UploadPart request
func (a *lowLevelAPI) uploadPartRequest(bucket, object, uploadID string, partNumber int, size int64, body io.ReadSeeker) (*request, error) {
	op := &operation{
		HTTPServer: a.config.MustGetEndpoint(),
		HTTPMethod: "PUT",
		HTTPPath:   "/" + bucket + "/" + object + "?partNumber=" + strconv.Itoa(partNumber) + "&uploadId=" + uploadID,
	}
	md5SumBytes, err := sumMD5Reader(body, size)
	if err != nil {
		return nil, err
	}
	r, err := newRequest(op, a.config, body)
	if err != nil {
		return nil, err
	}
	// set Content-MD5 as base64 encoded md5
	r.Set("Content-MD5", base64.StdEncoding.EncodeToString(md5SumBytes))
	r.req.ContentLength = size
	return r, nil
}

// uploadPart uploads a part in a multipart upload.
func (a *lowLevelAPI) uploadPart(bucket, object, uploadID string, partNumber int, size int64, body io.ReadSeeker) (*completePart, error) {
	req, err := a.uploadPartRequest(bucket, object, uploadID, partNumber, size, body)
	if err != nil {
		return nil, err
	}
	// get hex encoding for md5sum in base64
	md5SumBytes, err := base64.StdEncoding.DecodeString(req.Get("Content-MD5"))
	if err != nil {
		return nil, err
	}
	completePart := new(completePart)
	completePart.PartNumber = partNumber
	completePart.ETag = "\"" + hex.EncodeToString(md5SumBytes) + "\""

	// initiate the request
	resp, err := req.Do()
	if err != nil {
		return nil, err
	}
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return nil, responseToError(resp)
		}
	}
	return completePart, resp.Body.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/rhaamo/lutrainit/shared/ipc"
	"github.com/rhaamo/lutrainit/lutrainit/tools"
	"github.com/valyala/gorpc"
	"time"
	"runtime"
	"github.com/go-clog/clog"
)

var (
	startTime = time.Now()
)

func socketInitctl() {
	d := gorpc.NewDispatcher()

	// Returns the init daemon version
	d.AddFunc("version", func() *ipc.Version {
		return &ipc.Version{
			ServerVersion: LutraVersion,
			ServerBuildHash: LutraBuildGitHash,
			ServerBuildTime: LutraBuildTime,
		}
	})

	// Returns the daemon system stats
	d.AddFunc("stats", func() *ipc.SysStatus {
		return returnStats()
	})

	d.AddFunc("shutdown", func() {
		clog.Info("[lutra] I was asked to shutdown, goodbye!")
		doShutdown(false)
		// will never return, sorry
	})

	d.AddFunc("reboot", func() {
		clog.Info("[lutra] I was asked to reboot, seeya!")
		doShutdown(true)
		// will never return, sorry
	})

	// Returns processes statuses
	d.AddFunc("status", func(status *ipc.AskStatus) map[ipc.ServiceName]*ipc.Service {
		return returnStatus(status)
	})

	d.AddFunc("reload", func() *ipc.AnswerReload {
		err := ReloadConfig()
		if err != nil {
			return &ipc.AnswerReload{Err: true, ErrStr: err.Error()}
		}
		return &ipc.AnswerReload{Err: false}
	})

	s := gorpc.NewUnixServer("/run/ottersock", d.NewHandlerFunc())
	clog.Info("[lutra] RPC starting on socket: %s", s.Addr)

	if err := s.Serve(); err != nil {
		clog.Error(2, "[lutra][socket] Starting GoRPC error", err.Error())
		return
	}
	defer s.Stop()
}

func returnStats() *ipc.SysStatus {
	m := new(runtime.MemStats)
	runtime.ReadMemStats(m)

	return &ipc.SysStatus{
		Uptime: tools.TimeSincePro(startTime),

		NumGoroutine: runtime.NumGoroutine(),

		MemAllocated: tools.FileSize(int64(m.Alloc)),
		MemTotal: tools.FileSize(int64(m.TotalAlloc)),
		MemSys: tools.FileSize(int64(m.Sys)),
		Lookups: m.Lookups,
		MemMallocs: m.Mallocs,
		MemFrees: m.Frees,

		HeapAlloc: tools.FileSize(int64(m.HeapAlloc)),
		HeapSys: tools.FileSize(int64(m.HeapSys)),
		HeapIdle: tools.FileSize(int64(m.HeapIdle)),
		HeapInuse: tools.FileSize(int64(m.HeapInuse)),
		HeapReleased: tools.FileSize(int64(m.HeapReleased)),
		HeapObjects: m.HeapObjects,

		StackInuse: tools.FileSize(int64(m.StackInuse)),
		StackSys: tools.FileSize(int64(m.StackSys)),
		MSpanInuse: tools.FileSize(int64(m.MSpanInuse)),
		MSpanSys: tools.FileSize(int64(m.MSpanSys)),
		MCacheInuse: tools.FileSize(int64(m.MCacheInuse)),
		MCacheSys: tools.FileSize(int64(m.MCacheSys)),
		BuckHashSys: tools.FileSize(int64(m.BuckHashSys)),
		GCSys: tools.FileSize(int64(m.GCSys)),
		OtherSys: tools.FileSize(int64(m.OtherSys)),

		NextGC: tools.FileSize(int64(m.NextGC)),

		LastGC: fmt.Sprintf("%.1fs", float64(time.Now().UnixNano()-int64(m.LastGC))/1000/1000/1000),
		PauseTotalNs: fmt.Sprintf("%.1fs", float64(m.PauseTotalNs)/1000/1000/1000),
		PauseNs: fmt.Sprintf("%.3fs", float64(m.PauseNs[(m.NumGC+255)%256])/1000/1000/1000),
		NumGC: m.NumGC,
	}
}

func returnStatus(req *ipc.AskStatus) (services map[ipc.ServiceName]*ipc.Service) {
	services = make(map[ipc.ServiceName]*ipc.Service)

	if req.All {
		for k, v := range LoadedServices {
			services[ipc.ServiceName(k)] = &ipc.Service{
				Name: ipc.ServiceName(v.Name),
				Type: v.Type,
				Description: v.Description,
				State: ipc.RunState(v.State),
				LastKnownPID: v.LastKnownPID,
				LastAction: ipc.LastAction(v.LastAction),
				LastActionAt: v.LastActionAt,
				LastMessage: v.LastMessage,
			}
		}
	} else {
		if proc, exists := LoadedServices[ServiceName(req.Name)]; exists {
			services[ipc.ServiceName(proc.Name)] = &ipc.Service{
				Name:         ipc.ServiceName(proc.Name),
				Type:         proc.Type,
				Description:  proc.Description,
				State:        ipc.RunState(proc.State),
				LastKnownPID: proc.LastKnownPID,
				LastAction:   ipc.LastAction(proc.LastAction),
				LastActionAt: proc.LastActionAt,
				LastMessage:  proc.LastMessage,
			}
		} else {
			return nil
		}
	}

	return services
}<commit_msg>Missed that one<commit_after>package main

import (
	"fmt"
	"github.com/rhaamo/lutrainit/shared/ipc"
	"github.com/rhaamo/lutrainit/lutrainit/tools"
	"github.com/valyala/gorpc"
	"time"
	"runtime"
	"github.com/go-clog/clog"
)

var (
	startTime = time.Now()
)

func socketInitctl() {
	d := gorpc.NewDispatcher()

	// Returns the init daemon version
	d.AddFunc("version", func() *ipc.Version {
		return &ipc.Version{
			ServerVersion: LutraVersion,
			ServerBuildHash: LutraBuildGitHash,
			ServerBuildTime: LutraBuildTime,
		}
	})

	// Returns the daemon system stats
	d.AddFunc("stats", func() *ipc.SysStatus {
		return returnStats()
	})

	d.AddFunc("shutdown", func() {
		clog.Info("[lutra] I was asked to shutdown, goodbye!")
		doShutdown(false)
		// will never return, sorry
	})

	d.AddFunc("reboot", func() {
		clog.Info("[lutra] I was asked to reboot, seeya!")
		doShutdown(true)
		// will never return, sorry
	})

	// Returns processes statuses
	d.AddFunc("status", func(status *ipc.AskStatus) map[ipc.ServiceName]*ipc.Service {
		return returnStatus(status)
	})

	d.AddFunc("reload", func() *ipc.AnswerReload {
		err := ReloadConfig()
		if err != nil {
			return &ipc.AnswerReload{Err: true, ErrStr: err.Error()}
		}
		return &ipc.AnswerReload{Err: false}
	})

	s := gorpc.NewUnixServer("/run/ottersock", d.NewHandlerFunc())
	clog.Info("[lutra] RPC starting on socket: %s", s.Addr)

	if err := s.Serve(); err != nil {
		clog.Error(2, "[lutra][socket] Starting GoRPC error", err.Error())
		return
	}
	defer s.Stop()
}

func returnStats() *ipc.SysStatus {
	m := new(runtime.MemStats)
	runtime.ReadMemStats(m)

	return &ipc.SysStatus{
		Uptime: tools.TimeSincePro(startTime),

		NumGoroutine: runtime.NumGoroutine(),

		MemAllocated: tools.FileSize(int64(m.Alloc)),
		MemTotal: tools.FileSize(int64(m.TotalAlloc)),
		MemSys: tools.FileSize(int64(m.Sys)),
		Lookups: m.Lookups,
		MemMallocs: m.Mallocs,
		MemFrees: m.Frees,

		HeapAlloc: tools.FileSize(int64(m.HeapAlloc)),
		HeapSys: tools.FileSize(int64(m.HeapSys)),
		HeapIdle: tools.FileSize(int64(m.HeapIdle)),
		HeapInuse: tools.FileSize(int64(m.HeapInuse)),
		HeapReleased: tools.FileSize(int64(m.HeapReleased)),
		HeapObjects: m.HeapObjects,

		StackInuse: tools.FileSize(int64(m.StackInuse)),
		StackSys: tools.FileSize(int64(m.StackSys)),
		MSpanInuse: tools.FileSize(int64(m.MSpanInuse)),
		MSpanSys: tools.FileSize(int64(m.MSpanSys)),
		MCacheInuse: tools.FileSize(int64(m.MCacheInuse)),
		MCacheSys: tools.FileSize(int64(m.MCacheSys)),
		BuckHashSys: tools.FileSize(int64(m.BuckHashSys)),
		GCSys: tools.FileSize(int64(m.GCSys)),
		OtherSys: tools.FileSize(int64(m.OtherSys)),

		NextGC: tools.FileSize(int64(m.NextGC)),

		LastGC: fmt.Sprintf("%.1fs", float64(time.Now().UnixNano()-int64(m.LastGC))/1000/1000/1000),
		PauseTotalNs: fmt.Sprintf("%.1fs", float64(m.PauseTotalNs)/1000/1000/1000),
		PauseNs: fmt.Sprintf("%.3fs", float64(m.PauseNs[(m.NumGC+255)%256])/1000/1000/1000),
		NumGC: m.NumGC,
	}
}

func returnStatus(req *ipc.AskStatus) (services map[ipc.ServiceName]*ipc.Service) {
	services = make(map[ipc.ServiceName]*ipc.Service)

	if req.All {
		for k, v := range LoadedServices {
			services[ipc.ServiceName(k)] = &ipc.Service{
				Name: ipc.ServiceName(v.Name),
				Type: v.Type,
				Description: v.Description,
				State: ipc.RunState(v.State),
				LastKnownPID: v.LastKnownPID,
				LastAction: ipc.LastAction(v.LastAction),
				LastActionAt: v.LastActionAt,
				LastMessage: v.LastMessage,
				Deleted: v.Deleted,
			}
		}
	} else {
		if proc, exists := LoadedServices[ServiceName(req.Name)]; exists {
			services[ipc.ServiceName(proc.Name)] = &ipc.Service{
				Name:         ipc.ServiceName(proc.Name),
				Type:         proc.Type,
				Description:  proc.Description,
				State:        ipc.RunState(proc.State),
				LastKnownPID: proc.LastKnownPID,
				LastAction:   ipc.LastAction(proc.LastAction),
				LastActionAt: proc.LastActionAt,
				LastMessage:  proc.LastMessage,
				Deleted: proc.Deleted,
			}
		} else {
			return nil
		}
	}

	return services
}<|endoftext|>


<commit_before>package builder

import (
	"fmt"
	"os"

	log "github.com/sirupsen/logrus"
	rsync "github.com/zloylos/grsync"
)

// fetchSourceRsync copies from a local path to the destination.
func (b *Builder) fetchSourceRsync(destination, source string) error {
	st, err := os.Stat(source)
	if err != nil {
		return err
	}
	if !st.IsDir() {
		return fmt.Errorf("Cannot sync from %s, not a directory.", source)
	}
	log.WithField("source", source).WithField("destination", destination).Debug("Syncing")
	task := rsync.NewTask(source, destination, rsync.RsyncOptions{
		// Verbose increase verbosity
		Verbose: true,
		// Archve is archive mode; equals -rlptgoD (no -H,-A,-X)
		Archive: true,
		// Recurse into directories
		Recursive: true,
		// Links copy symlinks as symlinks
		Links: true,
	})
	return task.Run()
}
<commit_msg>fix: add / suffix to directory tree<commit_after>package builder

import (
	"fmt"
	"os"
	"strings"

	log "github.com/sirupsen/logrus"
	rsync "github.com/zloylos/grsync"
)

// fetchSourceRsync copies from a local path to the destination.
func (b *Builder) fetchSourceRsync(destination, source string) error {
	st, err := os.Stat(source)
	if err != nil {
		return err
	}
	if !st.IsDir() {
		return fmt.Errorf("Cannot sync from %s, not a directory.", source)
	}
	log.WithField("source", source).WithField("destination", destination).Debug("Syncing")
	if !strings.HasSuffix(destination, "/") {
		destination += "/"
	}
	if !strings.HasSuffix(source, "/") {
		source += "/"
	}
	task := rsync.NewTask(source, destination, rsync.RsyncOptions{
		// Verbose increase verbosity
		Verbose: true,
		// Archve is archive mode; equals -rlptgoD (no -H,-A,-X)
		Archive: true,
		// Recurse into directories
		Recursive: true,
		// Links copy symlinks as symlinks
		Links: true,
	})
	return task.Run()
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"io"
	"math"
	"strconv"
	"strings"
	"time"

	"github.com/jordic/goics"
	"github.com/lib/pq"
)

// round is taken from <https://github.com/golang/go/issues/4594#issuecomment-135336012>
func round(f float64) int {
	if math.Abs(f) < 0.5 {
		return 0
	}
	return int(f + math.Copysign(0.5, f))
}

// Event represents an ical event.
type Event struct {
	ID             int            `db:"id"`
	Start          time.Time      `db:"start_date"`
	End            time.Time      `db:"end_date"`
	AllDay         bool           `db:"all_day"`
	Title          string         `db:"title"`
	OrganizerName  string         `db:"organizer_name"`
	OrganizerEmail string         `db:"organizer_email"`
	Attendees      pq.StringArray `db:"attendees"`
	Location       string         `db:"location"`
	Recurrence     sql.NullString `db:"recurrence"`
	Resources      sql.NullString `db:"resources"`
	Comment        sql.NullString `db:"comment"`
	Type           sql.NullString `db:"type"`
}

func (e *Event) DayDuration() int {
	d := e.End.Sub(e.Start)
	return round(d.Hours() / 24)
}

// EventsCollection represents a collection of Event instances.
type EventsCollection []*Event

// EmitICal implements the ICalEmiter interface.
func (ec EventsCollection) EmitICal() goics.Componenter {
	c := goics.NewComponent()
	c.SetType("VCALENDAR")
	c.AddProperty("CALSCAL", "GREGORIAN")
	c.AddProperty("PRODID;X-RICAL-TZSOURCE=TZINFO", "-//geodata.soton.ac.uk")
	c.AddProperty("X-PUBLISHED-TTL", "PT1H") // Format: Duration ([RFC2445] section 4.3.6)
	c.AddProperty("METHOD", "PUBLISH")

	// Generate a component for each event.
	for _, ev := range ec {
		var desc, k, v string
		s := goics.NewComponent()
		s.SetType("VEVENT")
		s.AddProperty("SUMMARY", ev.Title)

		if ev.AllDay {
			k, v = goics.FormatDateField("DTSTART", ev.Start.In(time.Local))
			s.AddProperty(k, v)
			days := ev.DayDuration()
			if days > 1 {
				end := ev.Start.In(time.Local).Add(time.Duration(days) * (time.Hour * 24))
				k, v = goics.FormatDateField("DTEND", end)
				s.AddProperty(k, v)
			}
		} else {
			k, v = goics.FormatDateTimeField("DTSTART", ev.Start)
			s.AddProperty(k, v)
			k, v = goics.FormatDateTimeField("DTEND", ev.End)
			s.AddProperty(k, v)
		}

		s.AddProperty("UID", strconv.Itoa(ev.ID))

		if len(ev.Location) > 0 {
			s.AddProperty("LOCATION", ev.Location)
		}

		if ev.Type.Valid {
			cat := strings.Title(ev.Type.String)
			s.AddProperty("CATEGORIES", cat)
			desc += fmt.Sprintf("Category: %s\\n", cat)
		}

		if ev.Recurrence.Valid {
			desc += fmt.Sprintf("Recurrence: %s\\n", strings.Title(ev.Recurrence.String))
		}

		if ev.OrganizerEmail != "" {
			k = fmt.Sprintf("ORGANIZER;CN=%s", ev.OrganizerName)
			v = fmt.Sprintf("MAILTO:%s", ev.OrganizerEmail)
			s.AddProperty(k, v)
		}

		/* Attendees shouldn't appear in a PUBLISH calendar.
		for _, a := range ev.Attendees {
			k = fmt.Sprintf("ATTENDEE;CN=%s", a)
			s.AddProperty(k, "MAILTO:geodata@soton.ac.uk")
		}*/
		if len(ev.Attendees) > 0 {
			desc += fmt.Sprintf("Attendees: %s\\n", strings.Join(ev.Attendees, ", "))
		}

		if ev.Resources.Valid {
			s.AddProperty("RESOURCES", ev.Resources.String)
			desc += fmt.Sprintf("Resources: %s\\n", ev.Resources.String)
		}

		if ev.Comment.Valid {
			s.AddProperty("COMMENT", ev.Comment.String)
			desc += fmt.Sprintf("Comment: %s\\n", ev.Comment.String)
		}

		if len(desc) > 0 {
			s.AddProperty("DESCRIPTION", desc)
		}

		c.AddComponent(s)
	}

	return c
}

// Write encodes the events in iCalendar format to w.
func (ec EventsCollection) Write(w io.Writer) {
	goics.NewICalEncode(w).Encode(ec)
}
<commit_msg>Fix errors caused where the organiser email is `NULL`.<commit_after>package main

import (
	"database/sql"
	"fmt"
	"io"
	"math"
	"strconv"
	"strings"
	"time"

	"github.com/jordic/goics"
	"github.com/lib/pq"
)

// round is taken from <https://github.com/golang/go/issues/4594#issuecomment-135336012>
func round(f float64) int {
	if math.Abs(f) < 0.5 {
		return 0
	}
	return int(f + math.Copysign(0.5, f))
}

// Event represents an ical event.
type Event struct {
	ID             int            `db:"id"`
	Start          time.Time      `db:"start_date"`
	End            time.Time      `db:"end_date"`
	AllDay         bool           `db:"all_day"`
	Title          string         `db:"title"`
	OrganizerName  string         `db:"organizer_name"`
	OrganizerEmail sql.NullString `db:"organizer_email"`
	Attendees      pq.StringArray `db:"attendees"`
	Location       string         `db:"location"`
	Recurrence     sql.NullString `db:"recurrence"`
	Resources      sql.NullString `db:"resources"`
	Comment        sql.NullString `db:"comment"`
	Type           sql.NullString `db:"type"`
}

func (e *Event) DayDuration() int {
	d := e.End.Sub(e.Start)
	return round(d.Hours() / 24)
}

// EventsCollection represents a collection of Event instances.
type EventsCollection []*Event

// EmitICal implements the ICalEmiter interface.
func (ec EventsCollection) EmitICal() goics.Componenter {
	c := goics.NewComponent()
	c.SetType("VCALENDAR")
	c.AddProperty("CALSCAL", "GREGORIAN")
	c.AddProperty("PRODID;X-RICAL-TZSOURCE=TZINFO", "-//geodata.soton.ac.uk")
	c.AddProperty("X-PUBLISHED-TTL", "PT1H") // Format: Duration ([RFC2445] section 4.3.6)
	c.AddProperty("METHOD", "PUBLISH")

	// Generate a component for each event.
	for _, ev := range ec {
		var desc, k, v string
		s := goics.NewComponent()
		s.SetType("VEVENT")
		s.AddProperty("SUMMARY", ev.Title)

		if ev.AllDay {
			k, v = goics.FormatDateField("DTSTART", ev.Start.In(time.Local))
			s.AddProperty(k, v)
			days := ev.DayDuration()
			if days > 1 {
				end := ev.Start.In(time.Local).Add(time.Duration(days) * (time.Hour * 24))
				k, v = goics.FormatDateField("DTEND", end)
				s.AddProperty(k, v)
			}
		} else {
			k, v = goics.FormatDateTimeField("DTSTART", ev.Start)
			s.AddProperty(k, v)
			k, v = goics.FormatDateTimeField("DTEND", ev.End)
			s.AddProperty(k, v)
		}

		s.AddProperty("UID", strconv.Itoa(ev.ID))

		if len(ev.Location) > 0 {
			s.AddProperty("LOCATION", ev.Location)
		}

		if ev.Type.Valid {
			cat := strings.Title(ev.Type.String)
			s.AddProperty("CATEGORIES", cat)
			desc += fmt.Sprintf("Category: %s\\n", cat)
		}

		if ev.Recurrence.Valid {
			desc += fmt.Sprintf("Recurrence: %s\\n", strings.Title(ev.Recurrence.String))
		}

		if ev.OrganizerEmail.Valid && ev.OrganizerEmail.String != "" {
			k = fmt.Sprintf("ORGANIZER;CN=%s", ev.OrganizerName)
			v = fmt.Sprintf("MAILTO:%s", ev.OrganizerEmail.String)
			s.AddProperty(k, v)
		}

		/* Attendees shouldn't appear in a PUBLISH calendar.
		for _, a := range ev.Attendees {
			k = fmt.Sprintf("ATTENDEE;CN=%s", a)
			s.AddProperty(k, "MAILTO:geodata@soton.ac.uk")
		}*/
		if len(ev.Attendees) > 0 {
			desc += fmt.Sprintf("Attendees: %s\\n", strings.Join(ev.Attendees, ", "))
		}

		if ev.Resources.Valid {
			s.AddProperty("RESOURCES", ev.Resources.String)
			desc += fmt.Sprintf("Resources: %s\\n", ev.Resources.String)
		}

		if ev.Comment.Valid {
			s.AddProperty("COMMENT", ev.Comment.String)
			desc += fmt.Sprintf("Comment: %s\\n", ev.Comment.String)
		}

		if len(desc) > 0 {
			s.AddProperty("DESCRIPTION", desc)
		}

		c.AddComponent(s)
	}

	return c
}

// Write encodes the events in iCalendar format to w.
func (ec EventsCollection) Write(w io.Writer) {
	goics.NewICalEncode(w).Encode(ec)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"strings"

	mp "github.com/mackerelio/go-mackerel-plugin"
	"github.com/mackerelio/mackerel-agent/logging"
)

var logger = logging.GetLogger("metrics.plugin.jvm")

type JVMPlugin struct {
	Target    string
	Lvmid     string
	JstatPath string
	JavaName  string
	Tempfile  string
}

// # jps
// 26547 NettyServer
// 6438 Jps
func FetchLvmidByAppname(appname, target, jpsPath string) (string, error) {
	out, err := exec.Command(jpsPath, target).Output()
	if err != nil {
		logger.Errorf("Failed to run exec jps. %s", err)
		return "", err
	}
	for _, line := range strings.Split(string(out), "\n") {
		words := strings.Split(line, " ")
		if len(words) != 2 {
			continue
		}
		lvmid, name := words[0], words[1]
		if name == appname {
			return lvmid, nil
		}
	}
	return "", errors.New(fmt.Sprintf("Cannot get lvmid from %s", appname))
}

func fetchJstatMetrics(lvmid, option, jstatPath string) (map[string]float64, error) {
	out, err := exec.Command(jstatPath, option, lvmid).Output()
	if err != nil {
		logger.Errorf("Failed to run exec jstat. %s", err)
		return nil, err
	}

	lines := strings.Split(string(out), "\n")
	keys := strings.Fields(lines[0])
	values := strings.Fields(lines[1])

	stat := make(map[string]float64)
	for i, key := range keys {
		value, err := strconv.ParseFloat(values[i], 64)
		if err != nil {
			logger.Warningf("Failed to parse value. %s", err)
		}
		stat[key] = value
	}

	return stat, nil
}

func mergeStat(dst, src map[string]float64) {
	for k, v := range src {
		dst[k] = v
	}
}

// # jstat -gc <vmid>
//  S0C    S1C    S0U    S1U      EC       EU        OC         OU       PC     PU    YGC     YGCT    FGC    FGCT     GCT
// 3584.0 3584.0 2528.0  0.0   692224.0 19062.4  1398272.0   485450.1  72704.0 72611.3   3152   30.229   0      0.000   30.229

// # jstat -gccapacity  <vmid>
//  NGCMN    NGCMX     NGC     S0C   S1C       EC      OGCMN      OGCMX       OGC         OC      PGCMN    PGCMX     PGC       PC     YGC    FGC
// 699392.0 699392.0 699392.0 4096.0 4096.0 691200.0  1398272.0  1398272.0  1398272.0  1398272.0  21504.0 524288.0  72704.0  72704.0   4212     0

// # jstat -gcnew  <vmid>
//  S0C    S1C    S0U    S1U   TT MTT  DSS      EC       EU     YGC     YGCT
// 3072.0 3072.0    0.0 2848.0  1  15 3072.0 693248.0 626782.2   3463   33.658

func (m JVMPlugin) FetchMetrics() (map[string]float64, error) {
	gcStat, err := fetchJstatMetrics(m.Lvmid, "-gc", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcCapacityStat, err := fetchJstatMetrics(m.Lvmid, "-gccapacity", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcNewStat, err := fetchJstatMetrics(m.Lvmid, "-gcnew", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcOldStat, err := fetchJstatMetrics(m.Lvmid, "-gcold", m.JstatPath)
	if err != nil {
		return nil, err
	}

	stat := make(map[string]float64)
	mergeStat(stat, gcStat)
	mergeStat(stat, gcCapacityStat)
	mergeStat(stat, gcNewStat)
	mergeStat(stat, gcOldStat)

	return stat, nil
}

func (m JVMPlugin) GraphDefinition() map[string](mp.Graphs) {
	rawJavaName := m.JavaName
	lowerJavaName := strings.ToLower(m.JavaName)
	return map[string](mp.Graphs){
		fmt.Sprintf("jvm.%s.gc_events", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s GC events", rawJavaName),
			Unit:  "integer",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "YGC", Label: "Young GC event", Diff: true},
				mp.Metrics{Name: "FGC", Label: "Full GC event", Diff: true},
			},
		},
		fmt.Sprintf("jvm.%s.gc_time", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s GC time (msec)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "YGCT", Label: "Young GC time", Diff: true},
				mp.Metrics{Name: "FGCT", Label: "Full GC time", Diff: true},
			},
		},
		fmt.Sprintf("jvm.%s.new_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s New Space memory (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "NGCMX", Label: "New max", Diff: false},
				mp.Metrics{Name: "NGC", Label: "New current", Diff: false},
				mp.Metrics{Name: "EU", Label: "Eden used", Diff: false},
				mp.Metrics{Name: "S0U", Label: "Survivor0 used", Diff: false},
				mp.Metrics{Name: "S1U", Label: "Survivor1 used", Diff: false},
			},
		},
		fmt.Sprintf("jvm.%s.old_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s Old Space memory (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "OGCMX", Label: "Old max", Diff: false},
				mp.Metrics{Name: "OGC", Label: "Old current", Diff: false},
				mp.Metrics{Name: "OU", Label: "Old used", Diff: false},
			},
		},
		fmt.Sprintf("jvm.%s.perm_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s Permanent Space (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "PGCMX", Label: "Perm max", Diff: false},
				mp.Metrics{Name: "PGC", Label: "Perm current", Diff: false},
				mp.Metrics{Name: "PU", Label: "Perm used", Diff: false},
			},
		},
	}
}

func main() {
	optHost := flag.String("host", "localhost", "Hostname")
	optPort := flag.String("port", "1099", "Port")
	optJstatPath := flag.String("jstatpath", "/usr/bin/jstat", "jstat path")
	optJpsPath := flag.String("jpspath", "/usr/bin/jps", "jps path")
	optJavaName := flag.String("javaname", "", "Java app name")
	optPidFile := flag.String("pidfile", "", "pidfile path")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var jvm JVMPlugin
	jvm.Target = fmt.Sprintf("%s:%s", *optHost, *optPort)
	jvm.JstatPath = *optJstatPath

	if *optJavaName == "" {
		// https://docs.oracle.com/javase/7/docs/technotes/tools/share/jps.html
		// `The lvmid is typically, but not necessarily, the operating system's process identifier for the JVM process.`
		pid, err := ioutil.ReadFile(*optPidFile)
		if err != nil {
			logger.Errorf("Failed to load pid. %s", err)
			os.Exit(1)
		}
		jvm.Lvmid = string(pid)
	} else {
		lvmid, err := FetchLvmidByAppname(*optJavaName, jvm.Target, *optJpsPath)
		if err != nil {
			logger.Errorf("Failed to fetch lvmid. %s", err)
			os.Exit(1)
		}
		jvm.Lvmid = lvmid
	}

	jvm.JavaName = *optJavaName

	helper := mp.NewMackerelPlugin(jvm)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-jvm-%s", *optHost)
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<commit_msg>Use javaname option as a prefix of graph label if you use 'pidfile' option<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strconv"
	"strings"

	mp "github.com/mackerelio/go-mackerel-plugin"
	"github.com/mackerelio/mackerel-agent/logging"
)

var logger = logging.GetLogger("metrics.plugin.jvm")

type JVMPlugin struct {
	Target    string
	Lvmid     string
	JstatPath string
	JavaName  string
	Tempfile  string
}

// # jps
// 26547 NettyServer
// 6438 Jps
func FetchLvmidByAppname(appname, target, jpsPath string) (string, error) {
	out, err := exec.Command(jpsPath, target).Output()
	if err != nil {
		logger.Errorf("Failed to run exec jps. %s", err)
		return "", err
	}
	for _, line := range strings.Split(string(out), "\n") {
		words := strings.Split(line, " ")
		if len(words) != 2 {
			continue
		}
		lvmid, name := words[0], words[1]
		if name == appname {
			return lvmid, nil
		}
	}
	return "", errors.New(fmt.Sprintf("Cannot get lvmid from %s", appname))
}

func fetchJstatMetrics(lvmid, option, jstatPath string) (map[string]float64, error) {
	out, err := exec.Command(jstatPath, option, lvmid).Output()
	if err != nil {
		logger.Errorf("Failed to run exec jstat. %s", err)
		return nil, err
	}

	lines := strings.Split(string(out), "\n")
	keys := strings.Fields(lines[0])
	values := strings.Fields(lines[1])

	stat := make(map[string]float64)
	for i, key := range keys {
		value, err := strconv.ParseFloat(values[i], 64)
		if err != nil {
			logger.Warningf("Failed to parse value. %s", err)
		}
		stat[key] = value
	}

	return stat, nil
}

func mergeStat(dst, src map[string]float64) {
	for k, v := range src {
		dst[k] = v
	}
}

// # jstat -gc <vmid>
//  S0C    S1C    S0U    S1U      EC       EU        OC         OU       PC     PU    YGC     YGCT    FGC    FGCT     GCT
// 3584.0 3584.0 2528.0  0.0   692224.0 19062.4  1398272.0   485450.1  72704.0 72611.3   3152   30.229   0      0.000   30.229

// # jstat -gccapacity  <vmid>
//  NGCMN    NGCMX     NGC     S0C   S1C       EC      OGCMN      OGCMX       OGC         OC      PGCMN    PGCMX     PGC       PC     YGC    FGC
// 699392.0 699392.0 699392.0 4096.0 4096.0 691200.0  1398272.0  1398272.0  1398272.0  1398272.0  21504.0 524288.0  72704.0  72704.0   4212     0

// # jstat -gcnew  <vmid>
//  S0C    S1C    S0U    S1U   TT MTT  DSS      EC       EU     YGC     YGCT
// 3072.0 3072.0    0.0 2848.0  1  15 3072.0 693248.0 626782.2   3463   33.658

func (m JVMPlugin) FetchMetrics() (map[string]float64, error) {
	gcStat, err := fetchJstatMetrics(m.Lvmid, "-gc", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcCapacityStat, err := fetchJstatMetrics(m.Lvmid, "-gccapacity", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcNewStat, err := fetchJstatMetrics(m.Lvmid, "-gcnew", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcOldStat, err := fetchJstatMetrics(m.Lvmid, "-gcold", m.JstatPath)
	if err != nil {
		return nil, err
	}

	stat := make(map[string]float64)
	mergeStat(stat, gcStat)
	mergeStat(stat, gcCapacityStat)
	mergeStat(stat, gcNewStat)
	mergeStat(stat, gcOldStat)

	return stat, nil
}

func (m JVMPlugin) GraphDefinition() map[string](mp.Graphs) {
	rawJavaName := m.JavaName
	lowerJavaName := strings.ToLower(m.JavaName)
	return map[string](mp.Graphs){
		fmt.Sprintf("jvm.%s.gc_events", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s GC events", rawJavaName),
			Unit:  "integer",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "YGC", Label: "Young GC event", Diff: true},
				mp.Metrics{Name: "FGC", Label: "Full GC event", Diff: true},
			},
		},
		fmt.Sprintf("jvm.%s.gc_time", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s GC time (msec)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "YGCT", Label: "Young GC time", Diff: true},
				mp.Metrics{Name: "FGCT", Label: "Full GC time", Diff: true},
			},
		},
		fmt.Sprintf("jvm.%s.new_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s New Space memory (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "NGCMX", Label: "New max", Diff: false},
				mp.Metrics{Name: "NGC", Label: "New current", Diff: false},
				mp.Metrics{Name: "EU", Label: "Eden used", Diff: false},
				mp.Metrics{Name: "S0U", Label: "Survivor0 used", Diff: false},
				mp.Metrics{Name: "S1U", Label: "Survivor1 used", Diff: false},
			},
		},
		fmt.Sprintf("jvm.%s.old_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s Old Space memory (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "OGCMX", Label: "Old max", Diff: false},
				mp.Metrics{Name: "OGC", Label: "Old current", Diff: false},
				mp.Metrics{Name: "OU", Label: "Old used", Diff: false},
			},
		},
		fmt.Sprintf("jvm.%s.perm_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s Permanent Space (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "PGCMX", Label: "Perm max", Diff: false},
				mp.Metrics{Name: "PGC", Label: "Perm current", Diff: false},
				mp.Metrics{Name: "PU", Label: "Perm used", Diff: false},
			},
		},
	}
}

func main() {
	optHost := flag.String("host", "localhost", "Hostname")
	optPort := flag.String("port", "1099", "Port")
	optJstatPath := flag.String("jstatpath", "/usr/bin/jstat", "jstat path")
	optJpsPath := flag.String("jpspath", "/usr/bin/jps", "jps path")
	optJavaName := flag.String("javaname", "", "Java app name")
	optPidFile := flag.String("pidfile", "", "pidfile path")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var jvm JVMPlugin
	jvm.Target = fmt.Sprintf("%s:%s", *optHost, *optPort)
	jvm.JstatPath = *optJstatPath

	if *optJavaName == "" {
		logger.Errorf("javaname is required (if you use 'pidfile' option, 'javaname' is used as just a prefix of graph label)")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if *optPidFile == "" {
		// https://docs.oracle.com/javase/7/docs/technotes/tools/share/jps.html
		// `The lvmid is typically, but not necessarily, the operating system's process identifier for the JVM process.`
		pid, err := ioutil.ReadFile(*optPidFile)
		if err != nil {
			logger.Errorf("Failed to load pid. %s", err)
			os.Exit(1)
		}
		jvm.Lvmid = string(pid)
	} else {
		lvmid, err := FetchLvmidByAppname(*optJavaName, jvm.Target, *optJpsPath)
		if err != nil {
			logger.Errorf("Failed to fetch lvmid. %s", err)
			os.Exit(1)
		}
		jvm.Lvmid = lvmid
	}

	jvm.JavaName = *optJavaName

	helper := mp.NewMackerelPlugin(jvm)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-jvm-%s", *optHost)
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<|endoftext|>


<commit_before>package enamlbosh

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"

	"github.com/enaml-ops/enaml"
	"github.com/xchapter7x/lo"
)

// newRequest is like http.NewRequest, with the exception that it will add
// basic auth headers if the client is configured for basic auth.
func (s *Client) newRequest(method, url string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, url, body)
	if err == nil {
		req.SetBasicAuth(s.user, s.pass)
	}
	return req, err
}

func (s *Client) NewCloudConfigRequest(cloudconfig enaml.CloudConfigManifest) (req *http.Request, err error) {
	var b []byte
	var body io.Reader
	if b, err = cloudconfig.Bytes(); err == nil {
		body = bytes.NewReader(b)
		req, err = s.newRequest("POST", s.buildBoshURL("/cloud_configs"), body)
		req.Header.Set("content-type", "text/yaml")
	}
	return
}

func (s *Client) GetTask(taskID int, httpClient HttpClientDoer) (bt BoshTask, err error) {
	var req *http.Request
	var res *http.Response

	if req, err = s.newRequest("GET", s.buildBoshURL("/tasks/"+strconv.Itoa(taskID)), nil); err == nil {
		req.Header.Set("content-type", "text/yaml")

		if res, err = httpClient.Do(req); err == nil {
			defer res.Body.Close()
			lo.G.Debug("task request complete")
			var b []byte
			b, err = ioutil.ReadAll(res.Body)
			lo.G.Debug("rest resp: ", string(b))
			json.Unmarshal(b, &bt)
		}
	}

	if bt.ID != taskID && err == nil {
		err = fmt.Errorf("could not find the given task: %v", taskID)
	}
	return
}

func (s *Client) PostRemoteRelease(rls enaml.Release, httpClient HttpClientDoer) (bt BoshTask, err error) {

	if rls.URL == "" || rls.SHA1 == "" {
		err = fmt.Errorf("url or sha not set. these are required for remote stemcells URL: %s , SHA: %s", rls.URL, rls.SHA1)

	} else {
		var req *http.Request
		var res *http.Response
		var reqMap = map[string]string{
			"location": rls.URL,
			"sha1":     rls.SHA1,
		}
		var reqBytes, _ = json.Marshal(reqMap)
		var reqBody = bytes.NewReader(reqBytes)

		if req, err = s.newRequest("POST", s.buildBoshURL("/releases"), reqBody); err == nil {
			req.Header.Set("content-type", "application/json")

			if res, err = httpClient.Do(req); err == nil {
				defer res.Body.Close()
				lo.G.Debug("release request complete")
				var b []byte

				if b, err = ioutil.ReadAll(res.Body); err == nil {
					lo.G.Debug("rest resp: ", string(b))
					err = json.Unmarshal(b, &bt)
				}
			}
		}
	}
	return
}

func (s *Client) GetStemcells(httpClient HttpClientDoer) (stemcells []DeployedStemcell, err error) {
	var req *http.Request
	var res *http.Response

	if req, err = http.NewRequest("GET", s.buildBoshURL("/stemcells"), nil); err == nil {
		req.SetBasicAuth(s.user, s.pass)
		req.Header.Set("content-type", "application/json")

		if res, err = httpClient.Do(req); err == nil {
			defer res.Body.Close()
			var b []byte

			if b, err = ioutil.ReadAll(res.Body); err == nil {
				stemcells = make([]DeployedStemcell, 0)
				err = json.Unmarshal(b, &stemcells)
			}
		}
	}
	return stemcells, err
}

func (s *Client) CheckRemoteStemcell(sc enaml.Stemcell, httpClient HttpClientDoer) (exists bool, err error) {

	if (sc.Name == "" && sc.OS == "") || sc.Version == "" {
		err = fmt.Errorf("name or version not set. these are required to check for remote stemcells Name: %s , Version: %s", sc.Name, sc.Version)

	} else {
		exists = false

		if stemcells, err := s.GetStemcells(httpClient); err == nil {

			for _, stemcell := range stemcells {

				if (stemcell.Name == sc.Name || stemcell.OS == sc.OS) && stemcell.Version == sc.Version {
					exists = true
					break
				}
			}
		}
	}
	return
}

func (s *Client) PostRemoteStemcell(sc enaml.Stemcell, httpClient HttpClientDoer) (bt BoshTask, err error) {

	if sc.URL == "" || sc.SHA1 == "" {
		err = fmt.Errorf("url or sha not set. these are required for remote stemcells URL: %s , SHA: %s", sc.URL, sc.SHA1)

	} else {
		var req *http.Request
		var res *http.Response
		var reqMap = map[string]string{
			"location": sc.URL,
			"sha1":     sc.SHA1,
		}
		var reqBytes, _ = json.Marshal(reqMap)
		var reqBody = bytes.NewReader(reqBytes)

		if req, err = s.newRequest("POST", s.buildBoshURL("/stemcells"), reqBody); err == nil {
			req.Header.Set("content-type", "application/json")

			if res, err = httpClient.Do(req); err == nil {
				defer res.Body.Close()
				lo.G.Debug("stemcell request complete")
				var b []byte

				if b, err = ioutil.ReadAll(res.Body); err == nil {
					lo.G.Debug("rest resp: ", string(b))
					err = json.Unmarshal(b, &bt)
				}
			}
		}
	}
	return
}

func (s *Client) PostDeployment(deploymentManifest enaml.DeploymentManifest, httpClient HttpClientDoer) (boshTask BoshTask, err error) {
	var req *http.Request
	var res *http.Response
	var reqBody = bytes.NewReader(deploymentManifest.Bytes())

	if req, err = s.newRequest("POST", s.buildBoshURL("/deployments"), reqBody); err == nil {
		req.Header.Set("content-type", "text/yaml")

		if res, err = httpClient.Do(req); err == nil {
			defer res.Body.Close()
			lo.G.Debug("deployment request complete")
			var b []byte

			if b, err = ioutil.ReadAll(res.Body); err == nil {
				lo.G.Debug("rest resp: ", string(b))
				err = json.Unmarshal(b, &boshTask)
			}
		}
	}
	return
}

func (s *Client) GetCloudConfig(httpClient HttpClientDoer) (cloudconfig *enaml.CloudConfigManifest, err error) {
	var req *http.Request
	var res *http.Response
	var resBody = make([]CloudConfigResponseBody, 1)

	if req, err = s.newRequest("GET", s.buildBoshURL("/cloud_configs?limit=1"), nil); err == nil {
		req.Header.Set("content-type", "text/yaml")

		if res, err = httpClient.Do(req); err == nil {
			defer res.Body.Close()
			var b []byte
			b, err = ioutil.ReadAll(res.Body)
			lo.G.Debug("rest resp: ", string(b))
			json.Unmarshal(b, &resBody)
			cloudconfig = enaml.NewCloudConfigManifest([]byte(resBody[0].Properties))
		}
	}
	return
}

func (s *Client) GetInfo(httpClient HttpClientDoer) (bi *BoshInfo, err error) {
	var req *http.Request
	var res *http.Response
	bi = new(BoshInfo)

	if req, err = s.newRequest("GET", s.buildBoshURL("/info"), nil); err == nil {
		req.Header.Set("content-type", "text/yaml")

		if res, err = httpClient.Do(req); err == nil {
			var b []byte
			b, err = ioutil.ReadAll(res.Body)
			lo.G.Debug("rest resp: ", string(b))
			json.Unmarshal(b, bi)
		}
	}
	return
}

func (s *Client) buildBoshURL(urlpath string) (boshurl string) {
	boshurl = s.host + ":" + strconv.Itoa(s.port) + urlpath
	return
}
<commit_msg>[#126155341] clean up boshclient code to be a bit more idiomatic<commit_after>package enamlbosh

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"

	"github.com/enaml-ops/enaml"
	"github.com/xchapter7x/lo"
)

// newRequest is like http.NewRequest, with the exception that it will add
// basic auth headers if the client is configured for basic auth.
func (s *Client) newRequest(method, url string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, url, body)
	if err == nil {
		req.SetBasicAuth(s.user, s.pass)
	}
	return req, err
}

func (s *Client) NewCloudConfigRequest(cloudconfig enaml.CloudConfigManifest) (*http.Request, error) {
	b, err := cloudconfig.Bytes()
	if err != nil {
		return nil, err
	}
	body := bytes.NewReader(b)
	req, err := s.newRequest("POST", s.buildBoshURL("/cloud_configs"), body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("content-type", "text/yaml")
	return req, nil
}

func (s *Client) GetTask(taskID int, httpClient HttpClientDoer) (BoshTask, error) {
	req, err := s.newRequest("GET", s.buildBoshURL("/tasks/"+strconv.Itoa(taskID)), nil)
	if err != nil {
		return BoshTask{}, err
	}
	req.Header.Set("content-type", "text/yaml")
	res, err := httpClient.Do(req)
	if err != nil {
		return BoshTask{}, err
	}
	defer res.Body.Close()
	lo.G.Debug("task request complete")
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return BoshTask{}, err
	}
	lo.G.Debug("rest resp: ", string(b))
	var bt BoshTask
	json.Unmarshal(b, &bt)

	if bt.ID != taskID {
		return bt, fmt.Errorf("could not find the given task: %v", taskID)
	}
	return bt, nil
}

func (s *Client) PostRemoteRelease(rls enaml.Release, httpClient HttpClientDoer) (BoshTask, error) {
	if rls.URL == "" || rls.SHA1 == "" {
		return BoshTask{}, fmt.Errorf("url or sha not set. these are required for remote stemcells URL: %s , SHA: %s", rls.URL, rls.SHA1)
	}
	reqMap := map[string]string{
		"location": rls.URL,
		"sha1":     rls.SHA1,
	}
	reqBytes, _ := json.Marshal(reqMap)
	reqBody := bytes.NewReader(reqBytes)

	req, err := s.newRequest("POST", s.buildBoshURL("/releases"), reqBody)
	if err != nil {
		return BoshTask{}, err
	}

	req.Header.Set("content-type", "application/json")
	res, err := httpClient.Do(req)
	if err != nil {
		return BoshTask{}, err
	}

	defer res.Body.Close()
	lo.G.Debug("release request complete")

	var bt BoshTask
	err = json.NewDecoder(res.Body).Decode(&bt)
	return bt, err
}

func (s *Client) GetStemcells(httpClient HttpClientDoer) (stemcells []DeployedStemcell, err error) {
	req, err := s.newRequest("GET", s.buildBoshURL("/stemcells"), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("content-type", "application/json")
	res, err := httpClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	var stemcells []DeployedStemcell
	err = json.NewDecoder(res.Body).Decode(&stemcells)
	return stemcells, err
}

func (s *Client) CheckRemoteStemcell(sc enaml.Stemcell, httpClient HttpClientDoer) (exists bool, err error) {
	if (sc.Name == "" && sc.OS == "") || sc.Version == "" {
		return false, fmt.Errorf("name or version not set. these are required to check for remote stemcells Name: %s , Version: %s", sc.Name, sc.Version)
	}

	stemcells, err := s.GetStemcells(httpClient)
	if err != nil {
		return false, err
	}

	for _, stemcell := range stemcells {
		if (stemcell.Name == sc.Name || stemcell.OS == sc.OS) && stemcell.Version == sc.Version {
			return true, nil
		}
	}
	return false, nil
}

func (s *Client) PostRemoteStemcell(sc enaml.Stemcell, httpClient HttpClientDoer) (BoshTask, error) {
	if sc.URL == "" || sc.SHA1 == "" {
		return BoshTask{}, fmt.Errorf("url or sha not set. these are required for remote stemcells URL: %s , SHA: %s", sc.URL, sc.SHA1)
	}
	reqMap := map[string]string{
		"location": sc.URL,
		"sha1":     sc.SHA1,
	}
	reqBytes, _ := json.Marshal(reqMap)
	reqBody := bytes.NewReader(reqBytes)

	req, err := s.newRequest("POST", s.buildBoshURL("/stemcells"), reqBody)
	if err != nil {
		return BoshTask{}, err
	}

	req.Header.Set("content-type", "application/json")
	res, err := httpClient.Do(req)
	if err != nil {
		return BoshTask{}, err
	}

	defer res.Body.Close()
	lo.G.Debug("stemcell request complete")

	var bt BoshTask
	err = json.NewDecoder(res.Body).Decode(&bt)
	return bt, err
}

func (s *Client) PostDeployment(deploymentManifest enaml.DeploymentManifest, httpClient HttpClientDoer) (BoshTask, error) {
	reqBody := bytes.NewReader(deploymentManifest.Bytes())
	req, err := s.newRequest("POST", s.buildBoshURL("/deployments"), reqBody)
	if err != nil {
		return BoshTask{}, err
	}

	req.Header.Set("content-type", "text/yaml")
	res, err := httpClient.Do(req)
	if err != nil {
		return BoshTask{}, err
	}

	defer res.Body.Close()
	lo.G.Debug("deployment request complete")
	var bt BoshTask
	err = json.NewDecoder(res.Body).Decode(&bt)
	return bt, err
}

func (s *Client) GetCloudConfig(httpClient HttpClientDoer) (*enaml.CloudConfigManifest, error) {
	req, err := s.newRequest("GET", s.buildBoshURL("/cloud_configs?limit=1"), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("content-type", "text/yaml")
	res, err := httpClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	var cc []CloudConfigResponseBody
	err = json.NewDecoder(res.Body).Decode(&cc)
	if err != nil {
		return nil, err
	}
	return enaml.NewCloudConfigManifest([]byte(cc[0].Properties)), nil
}

func (s *Client) GetInfo(httpClient HttpClientDoer) (*BoshInfo, error) {
	req, err := s.newRequest("GET", s.buildBoshURL("/info"), nil)
	if err != nil {
		return nil, err
	}

	res, err := httpClient.Do(req)
	if err != nil {
		return nil, err
	}
	var bi BoshInfo
	err = json.NewDecoder(res.Body).Decode(&bi)
	if err != nil {
		return nil, err
	}
	return &bi, nil
}

func (s *Client) buildBoshURL(urlpath string) string {
	return s.host + ":" + strconv.Itoa(s.port) + urlpath
}
<|endoftext|>


<commit_before>package aws

import (
	"log"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/mitchellh/goamz/ec2"
)

// tagsSchema returns the schema to use for tags.
func tagsSchema() *schema.Schema {
	return &schema.Schema{
		Type:     schema.TypeMap,
		Optional: true,
		Computed: true,
	}
}

// setTags is a helper to set the tags for a resource. It expects the
// tags field to be named "tags"
func setTags(conn *ec2.EC2, d *schema.ResourceData) error {
	if d.HasChange("tags") {
		oraw, nraw := d.GetChange("tags")
		o := oraw.(map[string]interface{})
		n := nraw.(map[string]interface{})
		create, remove := diffTags(tagsFromMap(o), tagsFromMap(n))

		// Set tags
		if len(remove) > 0 {
			log.Printf("[DEBUG] Removing tags: %#v", remove)
			if _, err := conn.DeleteTags([]string{d.Id()}, remove); err != nil {
				return err
			}
		}
		if len(create) > 0 {
			log.Printf("[DEBUG] Creating tags: %#v", create)
			if _, err := conn.CreateTags([]string{d.Id()}, create); err != nil {
				return err
			}
		}
	}

	return nil
}

// diffTags takes our tags locally and the ones remotely and returns
// the set of tags that must be created, and the set of tags that must
// be destroyed.
func diffTags(oldTags, newTags []ec2.Tag) ([]ec2.Tag, []ec2.Tag) {
	// First, we're creating everything we have
	create := make(map[string]interface{})
	for _, t := range newTags {
		create[t.Key] = t.Value
	}

	// Build the list of what to remove
	var remove []ec2.Tag
	for _, t := range oldTags {
		old, ok := create[t.Key]
		if !ok || old != t.Value {
			// Delete it!
			remove = append(remove, t)
		}
	}

	return tagsFromMap(create), remove
}

// tagsFromMap returns the tags for the given map of data.
func tagsFromMap(m map[string]interface{}) []ec2.Tag {
	result := make([]ec2.Tag, 0, len(m))
	for k, v := range m {
		result = append(result, ec2.Tag{
			Key:   k,
			Value: v.(string),
		})
	}

	return result
}

// tagsToMap turns the list of tags into a map.
func tagsToMap(ts []ec2.Tag) map[string]string {
	result := make(map[string]string)
	for _, t := range ts {
		result[t.Key] = t.Value
	}

	return result
}
<commit_msg>"Computed" should be false<commit_after>package aws

import (
	"log"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/mitchellh/goamz/ec2"
)

// tagsSchema returns the schema to use for tags.
func tagsSchema() *schema.Schema {
	return &schema.Schema{
		Type:     schema.TypeMap,
		Optional: true,
	}
}

// setTags is a helper to set the tags for a resource. It expects the
// tags field to be named "tags"
func setTags(conn *ec2.EC2, d *schema.ResourceData) error {
	if d.HasChange("tags") {
		oraw, nraw := d.GetChange("tags")
		o := oraw.(map[string]interface{})
		n := nraw.(map[string]interface{})
		create, remove := diffTags(tagsFromMap(o), tagsFromMap(n))

		// Set tags
		if len(remove) > 0 {
			log.Printf("[DEBUG] Removing tags: %#v", remove)
			if _, err := conn.DeleteTags([]string{d.Id()}, remove); err != nil {
				return err
			}
		}
		if len(create) > 0 {
			log.Printf("[DEBUG] Creating tags: %#v", create)
			if _, err := conn.CreateTags([]string{d.Id()}, create); err != nil {
				return err
			}
		}
	}

	return nil
}

// diffTags takes our tags locally and the ones remotely and returns
// the set of tags that must be created, and the set of tags that must
// be destroyed.
func diffTags(oldTags, newTags []ec2.Tag) ([]ec2.Tag, []ec2.Tag) {
	// First, we're creating everything we have
	create := make(map[string]interface{})
	for _, t := range newTags {
		create[t.Key] = t.Value
	}

	// Build the list of what to remove
	var remove []ec2.Tag
	for _, t := range oldTags {
		old, ok := create[t.Key]
		if !ok || old != t.Value {
			// Delete it!
			remove = append(remove, t)
		}
	}

	return tagsFromMap(create), remove
}

// tagsFromMap returns the tags for the given map of data.
func tagsFromMap(m map[string]interface{}) []ec2.Tag {
	result := make([]ec2.Tag, 0, len(m))
	for k, v := range m {
		result = append(result, ec2.Tag{
			Key:   k,
			Value: v.(string),
		})
	}

	return result
}

// tagsToMap turns the list of tags into a map.
func tagsToMap(ts []ec2.Tag) map[string]string {
	result := make(map[string]string)
	for _, t := range ts {
		result[t.Key] = t.Value
	}

	return result
}
<|endoftext|>


<commit_before>package main

import (
	//"fmt"
	"image/color"
	"image/png"
	"os"
)

var (
	r1   = color.RGBA{153, 255, 255, 0}
	r5   = color.RGBA{103, 205, 255, 0}
	r10  = color.RGBA{30, 146, 255, 0}
	r20  = color.RGBA{0, 56, 255, 0}
	r30  = color.RGBA{250, 245, 0, 0}
	r50  = color.RGBA{255, 147, 0, 0}
	r80  = color.RGBA{252, 0, 0, 0}
	r100 = color.RGBA{154, 0, 121, 0}
)

func toInteger(rgba color.RGBA) (i int) {
	i = (int)(rgba.R)
	i <<= 8
	i += (int)(rgba.G)
	i <<= 8
	i += (int)(rgba.B)
	return
}

func toRGBA(c color.Color) (rgba color.RGBA) {
	r, g, b, _ := c.RGBA()
	rgba.R = (uint8)(r >> 8)
	rgba.G = (uint8)(g >> 8)
	rgba.B = (uint8)(b >> 8)
	rgba.A = 0
	//fmt.Printf("R:%d G:%d B:%d\n", rgba.R, rgba.G, rgba.B)
	return
}

func rgbToInteger(r, g, b uint8) (i int) {
	i = (int)(r)
	i <<= 8
	i += (int)(g)
	i <<= 8
	i += (int)(b)
	return
}

func limSet(v uint8, i int) uint8 {
	if (int)(v)+i < 0 {
		return 0
	} else if (int)(v)+i > 255 {
		return 255
	} else {
		return v + (uint8)(i)
	}
}

func appColSearch(c color.RGBA, rx color.RGBA) bool {
	r := rx.R
	g := rx.G
	b := rx.B
	ci := toInteger(c)
	//fmt.Printf("R:%d G:%d B:%d\n", c.R, c.G, c.B)

	for i := -APP_RANGE; i <= APP_RANGE; i++ {
		for j := -APP_RANGE; j <= APP_RANGE; j++ {
			for k := -APP_RANGE; k <= APP_RANGE; k++ {
				tr := limSet(r, i)
				tg := limSet(g, j)
				td := limSet(b, k)
				xi := rgbToInteger(tr, tg, td)
				if ci == xi {
					return true
				}
			}
		}
	}
	return false
}

func ColorCount() (int, error) {
	var w int
	f, err := os.Open(TRIM)
	if err != nil {
		return 0, err
	}
	defer f.Close()
	img, _ := png.Decode(f)
	bounds := img.Bounds()
	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			c := color.RGBAModel.Convert(img.At(x, y))
			rgba := toRGBA(c)
			if appColSearch(rgba, r1) {
				w += 1
			} else if appColSearch(rgba, r5) {
				w += 5
			} else if appColSearch(rgba, r10) {
				w += 10
			} else if appColSearch(rgba, r20) {
				w += 20
			} else if appColSearch(rgba, r30) {
				w += 30
			} else if appColSearch(rgba, r50) {
				w += 50
			} else if appColSearch(rgba, r100) {
				w += 100
			}
		}
	}

	return w, nil
}
<commit_msg>Fix Color Value #2<commit_after>package main

import (
	"fmt"
	"image/color"
	"image/png"
	"os"
)

var (
	r1   = color.RGBA{153, 255, 255, 0}
	r5   = color.RGBA{103, 205, 255, 0}
	r10  = color.RGBA{30, 146, 255, 0}
	r20  = color.RGBA{0, 56, 255, 0}
	r30  = color.RGBA{250, 245, 0, 0}
	r50  = color.RGBA{255, 153, 0, 0}
	r80  = color.RGBA{231, 40, 0, 0}
	r100 = color.RGBA{154, 0, 121, 0}
)

func toInteger(rgba color.RGBA) (i int) {
	i = (int)(rgba.R)
	i <<= 8
	i += (int)(rgba.G)
	i <<= 8
	i += (int)(rgba.B)
	return
}

func toRGBA(c color.Color) (rgba color.RGBA) {
	r, g, b, _ := c.RGBA()
	rgba.R = (uint8)(r >> 8)
	rgba.G = (uint8)(g >> 8)
	rgba.B = (uint8)(b >> 8)
	rgba.A = 0
	//fmt.Printf("R:%d G:%d B:%d\n", rgba.R, rgba.G, rgba.B)
	return
}

func rgbToInteger(r, g, b uint8) (i int) {
	i = (int)(r)
	i <<= 8
	i += (int)(g)
	i <<= 8
	i += (int)(b)
	return
}

func limSet(v uint8, i int) uint8 {
	if (int)(v)+i < 0 {
		return 0
	} else if (int)(v)+i > 255 {
		return 255
	} else {
		return v + (uint8)(i)
	}
}

func appColSearch(c color.RGBA, rx color.RGBA) bool {
	r := rx.R
	g := rx.G
	b := rx.B
	ci := toInteger(c)
	fmt.Printf("R:%d G:%d B:%d\n", c.R, c.G, c.B)

	for i := -APP_RANGE; i <= APP_RANGE; i++ {
		for j := -APP_RANGE; j <= APP_RANGE; j++ {
			for k := -APP_RANGE; k <= APP_RANGE; k++ {
				tr := limSet(r, i)
				tg := limSet(g, j)
				td := limSet(b, k)
				xi := rgbToInteger(tr, tg, td)
				if ci == xi {
					return true
				}
			}
		}
	}
	return false
}

func ColorCount() (int, error) {
	var w int
	f, err := os.Open(TRIM)
	if err != nil {
		return 0, err
	}
	defer f.Close()
	img, _ := png.Decode(f)
	bounds := img.Bounds()
	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			c := color.RGBAModel.Convert(img.At(x, y))
			rgba := toRGBA(c)
			if appColSearch(rgba, r1) {
				w += 1
			} else if appColSearch(rgba, r5) {
				w += 5
			} else if appColSearch(rgba, r10) {
				w += 10
			} else if appColSearch(rgba, r20) {
				w += 20
			} else if appColSearch(rgba, r30) {
				w += 30
			} else if appColSearch(rgba, r50) {
				w += 50
			} else if appColSearch(rgba, r100) {
				w += 100
			}
		}
	}

	return w, nil
}
<|endoftext|>


<commit_before>package command

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/nabeken/aaa/agent"
	"github.com/nabeken/aws-go-s3/bucket"
)

type LsCommand struct {
	S3Config *S3Config
	Email    string
	Format   string

	filer agent.Filer
}

func (c *LsCommand) init() {
	s3b := bucket.New(s3.New(session.New()), c.S3Config.Bucket)
	c.filer = agent.NewS3Filer(s3b, c.S3Config.KMSKeyID)
}

func (c *LsCommand) Run() error {
	c.init()

	output, err := c.FetchData()
	if err != nil {
		return err
	}

	switch c.Format {
	case "json":
		if err := json.NewEncoder(os.Stdout).Encode(output); err != nil {
			return err
		}
	default:
		fmt.Println("NOT IMPLEMENTED")
	}

	return nil
}

func (c *LsCommand) FetchData() (*aaadata, error) {
	data := aaadata{}

	accounts, err := c.ListAccounts()
	if err != nil {
		return nil, err
	}

	for _, a := range accounts {
		acc := account{
			Email: a,
		}

		store, err := agent.NewStore(a, c.filer)
		if err != nil {
			return nil, err
		}

		domains, err := store.ListDomains()
		if err != nil {
			return nil, err
		}

		for _, d := range domains {
			authz, err := store.LoadAuthorization(d)
			if err != nil {
				log.Printf("failed to load authorization for %s: %s. skipping...", d, err)
				continue
			}

			cert, err := store.LoadCert(d)
			if err != nil {
				log.Printf("failed to load certificate for %s: %s (or new-cert is ongoing). skipping...", d, err)
				continue
			}

			acc.Domains = append(acc.Domains, domain{
				Domain: d,
				Authorization: authorization{
					Expires: authz.GetExpires(),
				},
				Certificate: certificate{
					NotBefore: cert.NotBefore,
					NotAfter:  cert.NotAfter,
				},
			})
		}

		data.Accounts = append(data.Accounts, acc)
	}

	return &data, nil
}

func (c *LsCommand) ListAccounts() ([]string, error) {
	dirs, err := c.filer.ListDir(agent.StorePrefix)
	if err != nil {
		return nil, err
	}

	accounts := make([]string, 0, len(dirs))
	for _, dir := range dirs {
		elem := c.filer.Split(dir)

		// account (email) is in 2nd element
		if len(elem) > 1 {
			accounts = append(accounts, elem[1])
		}
	}

	return accounts, nil
}

type aaadata struct {
	Accounts []account `json:"accounts"`
}

type account struct {
	Email   string   `json:"email"`
	Domains []domain `json:"domains"`
}

type domain struct {
	Domain        string        `json:"domain"`
	Authorization authorization `json:"authorization"`
	Certificate   certificate   `json:"certificate"`
}

type authorization struct {
	Expires time.Time `json:"expires"`
}

type certificate struct {
	NotBefore time.Time `json:"not_before"`
	NotAfter  time.Time `json:"not_after"`
}
<commit_msg>ls: Improve data structure<commit_after>package command

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/nabeken/aaa/agent"
	"github.com/nabeken/aws-go-s3/bucket"
)

type LsCommand struct {
	S3Config *S3Config
	Email    string
	Format   string

	filer agent.Filer
}

func (c *LsCommand) init() {
	s3b := bucket.New(s3.New(session.New()), c.S3Config.Bucket)
	c.filer = agent.NewS3Filer(s3b, c.S3Config.KMSKeyID)
}

func (c *LsCommand) Run() error {
	c.init()

	output, err := c.FetchData()
	if err != nil {
		return err
	}

	switch c.Format {
	case "json":
		if err := json.NewEncoder(os.Stdout).Encode(output); err != nil {
			return err
		}
	default:
		fmt.Println("NOT IMPLEMENTED")
	}

	return nil
}

func (c *LsCommand) FetchData() ([]domain, error) {
	data := []domain{}

	emails, err := c.ListAccounts()
	if err != nil {
		return nil, err
	}

	for _, email := range emails {
		store, err := agent.NewStore(email, c.filer)
		if err != nil {
			return nil, err
		}

		domains, err := store.ListDomains()
		if err != nil {
			return nil, err
		}

		for _, dom := range domains {
			authz, err := store.LoadAuthorization(dom)
			if err != nil {
				log.Printf("failed to load authorization for %s: %s. skipping...", dom, err)
				continue
			}

			cert, err := store.LoadCert(dom)
			if err != nil {
				log.Printf("failed to load certificate for %s: %s (or new-cert is ongoing or this domain is in SAN in other certificates). skipping...", dom, err)
				continue
			}

			data = append(data, domain{
				Email:  email,
				Domain: dom,
				Authorization: authorization{
					Expires: authz.GetExpires(),
				},
				Certificate: certificate{
					NotBefore: cert.NotBefore,
					NotAfter:  cert.NotAfter,
					SAN:       cert.DNSNames,
				},
			})
		}
	}

	return data, nil
}

func (c *LsCommand) ListAccounts() ([]string, error) {
	dirs, err := c.filer.ListDir(agent.StorePrefix)
	if err != nil {
		return nil, err
	}

	accounts := make([]string, 0, len(dirs))
	for _, dir := range dirs {
		elem := c.filer.Split(dir)

		// account (email) is in 2nd element
		if len(elem) > 1 {
			accounts = append(accounts, elem[1])
		}
	}

	return accounts, nil
}

type domain struct {
	Email         string        `json:"email"`
	Domain        string        `json:"domain"`
	Authorization authorization `json:"authorization"`
	Certificate   certificate   `json:"certificate"`
}

type authorization struct {
	Expires time.Time `json:"expires"`
}

type certificate struct {
	NotBefore time.Time `json:"not_before"`
	NotAfter  time.Time `json:"not_after"`
	SAN       []string  `json:"san"`
}
<|endoftext|>