<commit_before>package god

import (
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Felamande/jsvm"
	"github.com/Felamande/otto"
	"github.com/demon-xxi/wildmatch"
	"gopkg.in/fsnotify.v1"
)

type taskRunner struct {
	wildcard string
	eventCb  otto.Value
	// errCb    otto.Value
	lastTime time.Time
	lastPath string
	// unique bool
}

var tasks []*taskRunner
var ignored []string
var wd string
var initCall []otto.Value
var once = new(sync.Once)

func callInit() {
	once.Do(func() {
		lenArg := len(initCall)
		if initCall != nil && lenArg > 0 {
			iarg := make([]interface{}, len(initCall)-1)

			for idx, arg := range initCall {
				if idx == 0 {
					continue
				}
				iarg[idx-1] = arg
			}
			jsvm.Callback(initCall[0], iarg...)
		}
	})

}

func init() {
	wd, _ = os.Getwd()
	wd = format(wd)
	if p := jsvm.Module("god"); p != nil {
		p.Extend("watch", watch)
		p.Extend("ignore", ignore)
		p.Extend("init", initfn)
	}
}

func initfn(call otto.FunctionCall) otto.Value {
	initCall = call.ArgumentList
	return otto.UndefinedValue()
}

func ignore(call otto.FunctionCall) otto.Value {
	for _, v := range call.ArgumentList {
		iarg, _ := v.Export()
		switch arg := iarg.(type) {
		case string:
			ignored = append(ignored, arg)
		case []string:
			ignored = append(ignored, arg...)
		}
	}
	return otto.UndefinedValue()
}

func watch(call otto.FunctionCall) otto.Value {
	wildcards, _ := call.Argument(0).Export()
	eventCb := call.Argument(1)
	switch w := wildcards.(type) {
	case string:
		tasks = append(tasks, &taskRunner{w, eventCb, time.Now(), ""})
	case []string:
		for _, wildcard := range w {
			tasks = append(tasks, &taskRunner{wildcard, eventCb, time.Now(), ""})
		}
	}

	return otto.UndefinedValue()
}

func Run() {
	callInit()

	w, _ := fsnotify.NewWatcher()
	filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
		for _, ig := range ignored {
			if wildmatch.IsSubsetOf(format(path), ig) {

				return filepath.SkipDir
			}
		}
		if !info.IsDir() {
			return nil
		}
		w.Add(path)

		return nil
	})
	for {
		select {
		case e := <-w.Events:
			rel := format(e.Name)
			abs := filepath.Join(wd, rel)
			if isDir(abs) {
				w.Add(rel)
			}

			for _, t := range tasks {

				if wildmatch.IsSubsetOf(path.Clean(rel), t.wildcard) {
					if t.lastPath == rel && time.Now().Sub(t.lastTime).Seconds() < 1 {
						goto end
					}
					jsvm.Callback(t.eventCb, abs, rel)
				end:
					t.lastPath = rel
					t.lastTime = time.Now()
				}

			}
		}
	}
	// return otto.UndefinedValue()

}
func format(path string) string {
	return strings.Replace(path, "\\", "/", -1)
}
func isDir(p string) bool {
	fi, err := os.Stat(p)
	return err == nil && fi.IsDir()
}
<commit_msg>unique task.<commit_after>package god

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Felamande/jsvm"
	"github.com/Felamande/otto"
	"github.com/demon-xxi/wildmatch"
	"gopkg.in/fsnotify.v1"
)

type taskRunner struct {
	wildcard string
	eventCb  otto.Value
	// errCb    otto.Value
	lastTime time.Time
	lastPath string
	unique   bool
}

var tasks []*taskRunner
var ignored []string
var wd string
var initCall []otto.Value
var once = new(sync.Once)

func callInit() {
	once.Do(func() {
		lenArg := len(initCall)
		if initCall != nil && lenArg > 0 {
			iarg := make([]interface{}, len(initCall)-1)

			for idx, arg := range initCall {
				if idx == 0 {
					continue
				}
				iarg[idx-1] = arg
			}
			jsvm.Callback(initCall[0], iarg...)
		}
	})

}

func init() {
	wd, _ = os.Getwd()
	wd = format(wd)
	if p := jsvm.Module("god"); p != nil {
		p.Extend("watch", watch)
		p.Extend("ignore", ignore)
		p.Extend("init", initfn)
	}
}

// function init(initfn, fnArgs...)
func initfn(call otto.FunctionCall) otto.Value {
	initCall = call.ArgumentList
	return otto.UndefinedValue()
}

func ignore(call otto.FunctionCall) otto.Value {
	for _, v := range call.ArgumentList {
		iarg, _ := v.Export()
		switch arg := iarg.(type) {
		case string:
			ignored = append(ignored, arg)
		case []string:
			ignored = append(ignored, arg...)
		}
	}
	return otto.UndefinedValue()
}

func watch(call otto.FunctionCall) otto.Value {
	wildcards, _ := call.Argument(0).Export()
	unique, _ := call.Argument(1).ToBoolean()
	eventCb := call.Argument(2)
	switch w := wildcards.(type) {
	case string:
		tasks = append(tasks, &taskRunner{w, eventCb, time.Now(), "", unique})
	case []string:
		for _, wildcard := range w {
			tasks = append(tasks, &taskRunner{wildcard, eventCb, time.Now(), "", unique})
		}
	}

	return otto.UndefinedValue()
}

func Run() {
	callInit()

	w, _ := fsnotify.NewWatcher()
	filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
		for _, ig := range ignored {
			if wildmatch.IsSubsetOf(format(path), ig) {

				return filepath.SkipDir
			}
		}
		if !info.IsDir() {
			return nil
		}
		w.Add(path)

		return nil
	})
	for {
		select {
		case e := <-w.Events:
			rel := format(e.Name)
			abs := filepath.Join(wd, rel)
			if isDir(abs) {
				w.Add(rel)
			}
			var uniqueTask *taskRunner
			var normalTasks []*taskRunner
			for _, t := range tasks {

				if wildmatch.IsSubsetOf(path.Clean(rel), t.wildcard) {
					if t.lastPath == rel && time.Now().Sub(t.lastTime).Seconds() < 1 {
						t.lastPath = rel
						t.lastTime = time.Now()
						continue
					}
					if t.unique {
						uniqueTask = t
					} else {
						normalTasks = append(normalTasks, t)
					}
					// jsvm.Callback(t.eventCb, abs, rel)
				}

			}
			if uniqueTask != nil {
				fmt.Println("I am unique")
				jsvm.Callback(uniqueTask.eventCb, abs, rel)
				uniqueTask.lastPath = rel
				uniqueTask.lastTime = time.Now()
				continue
			}
			for _, t := range normalTasks {
				fmt.Println("I am not unique")
				jsvm.Callback(t.eventCb, abs, rel)
				t.lastPath = rel
				t.lastTime = time.Now()
			}
		}
	}
	// return otto.UndefinedValue()

}
func format(path string) string {
	return strings.Replace(path, "\\", "/", -1)
}
func isDir(p string) bool {
	fi, err := os.Stat(p)
	return err == nil && fi.IsDir()
}
<|endoftext|>


<commit_before>package boardgame

import (
	"testing"
)

type stateComputeDelegate struct {
	testGameDelegate
	config *ComputedPropertiesConfig
}

func (s *stateComputeDelegate) ComputedPropertiesConfig() *ComputedPropertiesConfig {
	return s.config
}

func TestComputedPropertyDefinitionCompute(t *testing.T) {

	game := testGame()

	game.SetUp(0)

	var passedShadow *ShadowState

	definition := &ComputedPropertyDefinition{
		Dependencies: []StatePropertyRef{
			{
				Group:    StateGroupGame,
				PropName: "CurrentPlayer",
			},
			{
				Group:    StateGroupPlayer,
				PropName: "Score",
			},
		},
		PropType: TypeInt,
		Compute: func(shadow *ShadowState) (interface{}, error) {
			//For now we'll just pass it out for inspection
			passedShadow = shadow
			return nil, nil
		},
	}

	state := game.CurrentState()

	gameState, playerStates := concreteStates(state)

	gameState.CurrentPlayer = 5

	for i, playerState := range playerStates {
		playerState.Score = i + 1
	}

	definition.compute(state)

	if passedShadow == nil {
		t.Error("Calling compute on the rigged definition didn't set passedState")
	}

	if val, err := passedShadow.Game.IntProp("CurrentPlayer"); err != nil {
		t.Error("Unexpected error reading CurrentPlayer prop", err)
	} else if val != gameState.CurrentPlayer {
		t.Error("The shadow current player was not the real value. Got", val, "wanted", gameState.CurrentPlayer)
	}

	for i, playerState := range playerStates {
		playerShadow := passedShadow.Players[i]

		if val, err := playerShadow.IntProp("Score"); err != nil {
			t.Error("Unexpected error reading Score prop", err)
		} else if val != playerState.Score {
			t.Error("Unexpected score was not real value. Got", val, "wanted", playerState.Score)
		}
	}

}

func TestStateComputed(t *testing.T) {

	delegate := &stateComputeDelegate{}

	manager := NewGameManager(delegate, newTestGameChest(), newTestStorageManager())

	manager.SetUp()

	game := NewGame(manager)

	game.SetUp(0)

	state := game.CurrentState()

	gameState, playerStates := concreteStates(state)

	gameState.CurrentPlayer = 4

	playerStates[0].Score = 10
	playerStates[1].Score = 5

	config := &ComputedPropertiesConfig{
		Properties: map[string]ComputedPropertyDefinition{
			"CurrentPlayerPlusFive": ComputedPropertyDefinition{
				Dependencies: []StatePropertyRef{
					{
						Group:    StateGroupGame,
						PropName: "CurrentPlayer",
					},
				},
				PropType: TypeInt,
				Compute: func(shadow *ShadowState) (interface{}, error) {
					val, err := shadow.Game.IntProp("CurrentPlayer")
					if err != nil {
						return nil, err
					}
					return val + 5, nil
				},
			},
			"SumAllScores": ComputedPropertyDefinition{
				Dependencies: []StatePropertyRef{
					{
						Group:    StateGroupPlayer,
						PropName: "Score",
					},
				},
				PropType: TypeInt,
				Compute: func(shadow *ShadowState) (interface{}, error) {
					result := 0
					for _, player := range shadow.Players {
						val, err := player.IntProp("Score")

						if err != nil {
							return nil, err
						}

						result += val
					}
					return result, nil
				},
			},
		},
	}

	delegate.config = config

	computed := state.Computed()

	if val, err := computed.IntProp("CurrentPlayerPlusFive"); err != nil {
		t.Error("Unexpected error retrieving CurrentPlayerPlusFive", err)
	} else {
		if val != 4+5 {
			t.Error("CurrentPlayerPlusFive was unexpected value. Wanted", 4+5, "got", val)
		}
	}

	if val, err := computed.IntProp("SumAllScores"); err != nil {
		t.Error("Unexpected error retrieving SumAllScores", err)
	} else if val != 15 {
		t.Error("Unexpected result for SumAllScores. Got", val, "wanted", 15)
	}

	if _, err := computed.BoolProp("Foo"); err == nil {
		t.Error("Didn't get an error reading an unexpected bool prop")
	}

}
<commit_msg>One of the tests now check that game successfully set up. We really should have this test everywhere...<commit_after>package boardgame

import (
	"testing"
)

type stateComputeDelegate struct {
	testGameDelegate
	config *ComputedPropertiesConfig
}

func (s *stateComputeDelegate) ComputedPropertiesConfig() *ComputedPropertiesConfig {
	return s.config
}

func TestComputedPropertyDefinitionCompute(t *testing.T) {

	game := testGame()

	if err := game.SetUp(0); err != nil {
		t.Fatal("Game failed to set up", err)
	}

	var passedShadow *ShadowState

	definition := &ComputedPropertyDefinition{
		Dependencies: []StatePropertyRef{
			{
				Group:    StateGroupGame,
				PropName: "CurrentPlayer",
			},
			{
				Group:    StateGroupPlayer,
				PropName: "Score",
			},
		},
		PropType: TypeInt,
		Compute: func(shadow *ShadowState) (interface{}, error) {
			//For now we'll just pass it out for inspection
			passedShadow = shadow
			return nil, nil
		},
	}

	state := game.CurrentState()

	gameState, playerStates := concreteStates(state)

	gameState.CurrentPlayer = 5

	for i, playerState := range playerStates {
		playerState.Score = i + 1
	}

	definition.compute(state)

	if passedShadow == nil {
		t.Error("Calling compute on the rigged definition didn't set passedState")
	}

	if val, err := passedShadow.Game.IntProp("CurrentPlayer"); err != nil {
		t.Error("Unexpected error reading CurrentPlayer prop", err)
	} else if val != gameState.CurrentPlayer {
		t.Error("The shadow current player was not the real value. Got", val, "wanted", gameState.CurrentPlayer)
	}

	for i, playerState := range playerStates {
		playerShadow := passedShadow.Players[i]

		if val, err := playerShadow.IntProp("Score"); err != nil {
			t.Error("Unexpected error reading Score prop", err)
		} else if val != playerState.Score {
			t.Error("Unexpected score was not real value. Got", val, "wanted", playerState.Score)
		}
	}

}

func TestStateComputed(t *testing.T) {

	delegate := &stateComputeDelegate{}

	manager := NewGameManager(delegate, newTestGameChest(), newTestStorageManager())

	manager.SetUp()

	game := NewGame(manager)

	game.SetUp(0)

	state := game.CurrentState()

	gameState, playerStates := concreteStates(state)

	gameState.CurrentPlayer = 4

	playerStates[0].Score = 10
	playerStates[1].Score = 5

	config := &ComputedPropertiesConfig{
		Properties: map[string]ComputedPropertyDefinition{
			"CurrentPlayerPlusFive": ComputedPropertyDefinition{
				Dependencies: []StatePropertyRef{
					{
						Group:    StateGroupGame,
						PropName: "CurrentPlayer",
					},
				},
				PropType: TypeInt,
				Compute: func(shadow *ShadowState) (interface{}, error) {
					val, err := shadow.Game.IntProp("CurrentPlayer")
					if err != nil {
						return nil, err
					}
					return val + 5, nil
				},
			},
			"SumAllScores": ComputedPropertyDefinition{
				Dependencies: []StatePropertyRef{
					{
						Group:    StateGroupPlayer,
						PropName: "Score",
					},
				},
				PropType: TypeInt,
				Compute: func(shadow *ShadowState) (interface{}, error) {
					result := 0
					for _, player := range shadow.Players {
						val, err := player.IntProp("Score")

						if err != nil {
							return nil, err
						}

						result += val
					}
					return result, nil
				},
			},
		},
	}

	delegate.config = config

	computed := state.Computed()

	if val, err := computed.IntProp("CurrentPlayerPlusFive"); err != nil {
		t.Error("Unexpected error retrieving CurrentPlayerPlusFive", err)
	} else {
		if val != 4+5 {
			t.Error("CurrentPlayerPlusFive was unexpected value. Wanted", 4+5, "got", val)
		}
	}

	if val, err := computed.IntProp("SumAllScores"); err != nil {
		t.Error("Unexpected error retrieving SumAllScores", err)
	} else if val != 15 {
		t.Error("Unexpected result for SumAllScores. Got", val, "wanted", 15)
	}

	if _, err := computed.BoolProp("Foo"); err == nil {
		t.Error("Didn't get an error reading an unexpected bool prop")
	}

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"io/ioutil"
	"os"
	"os/exec"
	"syscall"
)

func pipe(data []byte) {
	flag.Parse()

	cmd := exec.Command("jq", flag.Args()...)
	cmd.Stdin = bytes.NewReader(data)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	if err := cmd.Start(); err != nil {
		panic(err)
	}
	if err := cmd.Wait(); err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				os.Exit(status.ExitStatus())
			}
		}
	}
}

func main() {
	data, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		panic(err)
	}
	decoder := json.NewDecoder(bytes.NewReader(data))
	for decoder.More() {
		var obj interface{}
		err := decoder.Decode(&obj)
		if err != nil {
			goto dump
		}
	}
	pipe(data)
	return

dump:
	os.Stdout.Write(data)
}
<commit_msg>Parse args correctly in `jqtxt`<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"syscall"
)

func pipe(data []byte) {
	var args []string
	if len(os.Args) > 1 {
		args = os.Args[1:]
	} else {
		args = []string{"."}
	}

	cmd := exec.Command("jq", args...)
	cmd.Stdin = bytes.NewReader(data)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	if err := cmd.Start(); err != nil {
		panic(err)
	}
	if err := cmd.Wait(); err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				os.Exit(status.ExitStatus())
			}
		}
	}
}

func main() {
	data, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		panic(err)
	}
	decoder := json.NewDecoder(bytes.NewReader(data))
	for decoder.More() {
		var obj interface{}
		err := decoder.Decode(&obj)
		if err != nil {
			goto dump
		}
	}
	pipe(data)
	return

dump:
	os.Stdout.Write(data)
}
<|endoftext|>


<commit_before>package m_etcd_test

import (
	"errors"
	"testing"
	"time"

	"github.com/coreos/go-etcd/etcd"
	"github.com/lytics/metafora"
	"github.com/lytics/metafora/m_etcd"
	"github.com/lytics/metafora/m_etcd/testutil"
	"github.com/lytics/metafora/statemachine"
)

const recursive = true

// TestSleepTest is an integration test for all of m_etcd's components.
//
func TestSleepTest(t *testing.T) {
	etcdc, hosts := testutil.NewEtcdClient(t)
	t.Parallel()
	const namespace = "sleeptest-metafora"
	const sleepingtasks = "sleeping-task1"

	etcdc.Delete(namespace, recursive)

	holdtask := make(chan bool)
	h := func(task metafora.Task, cmds <-chan *statemachine.Message) *statemachine.Message {

		if task.ID() == sleepingtasks {
			sleeptil := 5 * time.Second
			nextstarttime := (time.Now().Add(sleeptil))
			t.Logf("sleeping task:%v sleepfor:%v", task, nextstarttime)
			<-holdtask
			return statemachine.SleepMessage(nextstarttime)
		}

		cmd := <-cmds
		t.Logf("non sleeping task:%v", task)

		return cmd
	}

	newC := func(name, ns string) *metafora.Consumer {
		conf := m_etcd.NewConfig(ns, hosts)
		conf.Name = name
		coord, hf, bal, err := m_etcd.New(conf, h)
		if err != nil {
			t.Fatalf("Error creating new etcd stack: %v", err)
		}
		cons, err := metafora.NewConsumer(coord, hf, bal)
		if err != nil {
			t.Fatalf("Error creating consumer %s:%s: %v", ns, name, err)
		}
		go func() {
			cons.Run()
			t.Logf("Consumer:%s exited.", name)
		}()
		return cons
	}

	assertRunning := func(tid string, cons ...*metafora.Consumer) {
		found := false
		for _, c := range cons {
			tasks := c.Tasks()
			if len(tasks) > 0 && found {
				t.Fatal("Task already found running but another task is running on a different consumer")
			}
			if len(tasks) > 1 {
				t.Fatalf("Expected at most 1 task, but found: %d", len(tasks))
			}
			if len(tasks) == 1 && tasks[0].Task().ID() == tid {
				found = true
			}
		}
		if !found {
			t.Fatalf("Could not find task=%q", tid)
		}
	}

	// Start 2 consumers
	cons1 := newC("node1", namespace)
	cons2 := newC("node2", namespace)

	// Create clients and start some tests
	cliA := m_etcd.NewClient(namespace, hosts)

	if err := cliA.SubmitTask(m_etcd.DefaultTaskFunc(sleepingtasks, "")); err != nil {
		t.Fatalf("Error submitting task1 to a: %v", err)
	}

	// Give consumers a bit to pick up tasks
	time.Sleep(500 * time.Millisecond)

	assertRunning(sleepingtasks, cons1, cons2)

	holdtask <- true
	// Give consumers a bit to pick up tasks
	time.Sleep(500 * time.Millisecond)

	assertRunning(sleepingtasks, cons1, cons2) // not sure if this should be true or false.

	wait1 := make(chan bool)
	go func() {
		defer close(wait1)
		// Shutdown
		cons1.Shutdown()
		cons2.Shutdown()
	}()

	timeout := time.NewTimer(1 * time.Second)
	select {
	case <-wait1:
	case <-timeout.C:
		t.Fatalf("failed waiting for shutdown")
	}

	//	make sure all tasks are released
	for _, c := range []*metafora.Consumer{cons1, cons2} {
		tasks := c.Tasks()
		for _, work := range tasks {
			t.Fatalf("work id %v is still running", work)
		}
	}
}

// TestAll is an integration test for all of m_etcd's components.
//
// While huge integration tests like this are rarely desirable as they can be
// overly fragile and complex, I found myself manually repeating the tests I've
// automated here over and over. This is far more reliable than expecting
// developers to do adhoc testing of all of the m_etcd package's features.
func TestAll(t *testing.T) {
	etcdc, hosts := testutil.NewEtcdClient(t)
	t.Parallel()

	etcdc.Delete("test-a", recursive)
	etcdc.Delete("test-b", recursive)

	h := func(task metafora.Task, cmds <-chan *statemachine.Message) *statemachine.Message {
		cmd := <-cmds
		if task.ID() == "error-test" {
			return statemachine.ErrorMessage(errors.New("error-test"))
		}
		return cmd
	}

	newC := func(name, ns string) *metafora.Consumer {
		conf := m_etcd.NewConfig(ns, hosts)
		conf.Name = name
		coord, hf, bal, err := m_etcd.New(conf, h)
		if err != nil {
			t.Fatalf("Error creating new etcd stack: %v", err)
		}
		cons, err := metafora.NewConsumer(coord, hf, bal)
		if err != nil {
			t.Fatalf("Error creating consumer %s:%s: %v", ns, name, err)
		}
		go cons.Run()
		return cons
	}
	// Start 4 consumers, 2 per namespace
	cons1a := newC("node1", "test-a")
	cons2a := newC("node2", "test-a")
	cons1b := newC("node1", "test-b")
	cons2b := newC("node2", "test-b")

	// Create clients and start some tests
	cliA := m_etcd.NewClient("test-a", hosts)
	cliB := m_etcd.NewClient("test-b", hosts)

	if err := cliA.SubmitTask(m_etcd.DefaultTaskFunc("task1", "")); err != nil {
		t.Fatalf("Error submitting task1 to a: %v", err)
	}
	if err := cliB.SubmitTask(m_etcd.DefaultTaskFunc("task1", "")); err != nil {
		t.Fatalf("Error submitting task1 to b: %v", err)
	}

	// Give consumers a bit to pick up tasks
	time.Sleep(250 * time.Millisecond)

	assertRunning := func(tid string, cons ...*metafora.Consumer) {
		found := false
		for _, c := range cons {
			tasks := c.Tasks()
			if len(tasks) > 0 && found {
				t.Fatal("Task already found running but another task is running on a different consumer")
			}
			if len(tasks) > 1 {
				t.Fatalf("Expected at most 1 task, but found: %d", len(tasks))
			}
			if len(tasks) == 1 && tasks[0].Task().ID() == tid {
				found = true
			}
		}
		if !found {
			t.Fatalf("Could not find task=%q", tid)
		}
	}

	assertRunning("task1", cons1a, cons2a)
	assertRunning("task1", cons1b, cons2b)

	// Kill task1 in A
	{
		cmdr := m_etcd.NewCommander("test-a", etcdc)
		if err := cmdr.Send("task1", statemachine.KillMessage()); err != nil {
			t.Fatalf("Error sending kill to task1: %v", err)
		}
		time.Sleep(250 * time.Millisecond)

		for _, c := range []*metafora.Consumer{cons1a, cons2a} {
			tasks := c.Tasks()
			if len(tasks) != 0 {
				t.Fatalf("Expected no tasks but found: %d", len(tasks))
			}
		}
	}

	// Submit a bunch of tasks to A
	{
		tasks := []string{"task2", "task3", "task4", "task5", "task6", "task7"}
		for _, tid := range tasks {
			if err := cliA.SubmitTask(m_etcd.DefaultTaskFunc(tid, "")); err != nil {
				t.Fatalf("Error submitting task=%q to A: %v", tid, err)
			}
		}

		// Give them time to start
		time.Sleep(800 * time.Millisecond)

		// Ensure they're balanced
		if err := cliA.SubmitCommand("node1", metafora.CommandBalance()); err != nil {
			t.Fatalf("Error submitting balance command to cons1a: %v", err)
		}
		time.Sleep(800 * time.Millisecond)
		if err := cliA.SubmitCommand("node2", metafora.CommandBalance()); err != nil {
			t.Fatalf("Error submitting balance command to cons1a: %v", err)
		}

		a1tasks := cons1a.Tasks()
		a2tasks := cons2a.Tasks()
		for _, task := range a1tasks {
			metafora.Debug("A1: ", task.Task(), " - ", task.Stopped().IsZero())
		}
		for _, task := range a2tasks {
			metafora.Debug("A2: ", task.Task(), " - ", task.Stopped().IsZero())
		}
		time.Sleep(800 * time.Millisecond)

		a1tasks = cons1a.Tasks()
		a2tasks = cons2a.Tasks()
		if len(a1tasks) < 2 || len(a1tasks) > 4 || len(a2tasks) < 2 || len(a2tasks) > 4 {
			t.Fatalf("Namespace A isn't fairly balanced: node1: %d; node2: %d", len(a1tasks), len(a2tasks))
		}

		// Shutting down a consumer should migrate all tasks to the other
		cons1a.Shutdown()
		time.Sleep(800 * time.Millisecond)

		a2tasks = cons2a.Tasks()
		if len(a2tasks) != len(tasks) {
			t.Fatalf("Consumer 2a should have received all %d tasks but only has %d.", len(tasks), len(a2tasks))
		}
	}

	// Use Namespace B to check Error state handling
	{
		tasks := []string{"task8", "error-test"}
		for _, tid := range tasks {
			if err := cliB.SubmitTask(m_etcd.DefaultTaskFunc(tid, "")); err != nil {
				t.Fatalf("Error submitting task=%q to B: %v", tid, err)
			}
		}

		// Give them time to start
		time.Sleep(500 * time.Millisecond)

		n := len(cons1b.Tasks()) + len(cons2b.Tasks())
		if n != 3 {
			t.Fatalf("Expected B to be running 3 tasks but found %d", n)
		}

		// Resuming error-test 8*2 times should cause it to be failed
		cmdr := m_etcd.NewCommander("test-b", etcdc)
		for i := 0; i < statemachine.DefaultErrMax*2; i++ {
			if err := cmdr.Send("error-test", statemachine.RunMessage()); err != nil {
				t.Fatalf("Unexpected error resuming error-test in B: %v", err)
			}
			time.Sleep(200 * time.Millisecond)
		}

		n = len(cons1b.Tasks()) + len(cons2b.Tasks())
		if n != 2 {
			t.Fatalf("Expected B to be running 2 tasks but found %d", n)
		}

		// Resubmitting a failed task shouldn't error but also shouldn't run.
		if err := cliB.SubmitTask(m_etcd.DefaultTaskFunc("error-test", "")); err != nil {
			t.Fatalf("Error resubmitting error-test task to B: %v", err)
		}

		// Give the statemachine a moment to load the initial state and exit
		time.Sleep(200 * time.Millisecond)

		n = len(cons1b.Tasks()) + len(cons2b.Tasks())
		if n != 2 {
			t.Fatalf("Expected B to be running 2 tasks but found %d", n)
		}
	}

	// Shutdown
	cons2a.Shutdown()
	cons1b.Shutdown()
	cons2b.Shutdown()

	// Make sure everything is cleaned up
	respA, err := etcdc.Get("/test-a/tasks", true, true)
	if err != nil {
		t.Fatalf("Error getting tasks from etcd: %v", err)
	}
	respB, err := etcdc.Get("/test-b/tasks", true, true)
	if err != nil {
		t.Fatalf("Error getting tasks from etcd: %v", err)
	}

	nodes := []*etcd.Node{}
	nodes = append(nodes, respA.Node.Nodes...)
	nodes = append(nodes, respB.Node.Nodes...)
	for _, node := range nodes {
		if len(node.Nodes) > 1 {
			t.Fatalf("%s has %d (>1) nodes. First 2: %s, %s", node.Key, len(node.Nodes), node.Nodes[0].Key, node.Nodes[1].Key)
		}
	}
}

// TestTaskResurrectionInt ensures that a Claim won't recreate a task that had
// been deleted (marked as done). taskmgr has a non-integration version of this
// test.
func TestTaskResurrectionInt(t *testing.T) {
	etcdc, hosts := testutil.NewEtcdClient(t)
	t.Parallel()

	etcdc.Delete("test-resurrect", recursive)

	task := m_etcd.DefaultTaskFunc("xyz", "")

	conf := m_etcd.NewConfig("test-resurrect", hosts)
	coord, err := m_etcd.NewEtcdCoordinator(conf)
	if err != nil {
		t.Fatalf("Error creating coordinator: %v", err)
	}
	if err := coord.Init(nil); err != nil {
		t.Fatalf("Error initializing coordinator: %v", err)
	}

	// Try to claim a nonexistent
	if claimed := coord.Claim(task); claimed {
		t.Fatal("Claiming a nonexistent task should not work but did!")
	}

	// Create a task, mark it as done, and try to claim it again
	client := m_etcd.NewClient("test-resurrect", hosts)
	if err := client.SubmitTask(m_etcd.DefaultTaskFunc("xyz", "")); err != nil {
		t.Fatalf("Error submitting task xyz: %v", err)
	}

	if claimed := coord.Claim(task); !claimed {
		t.Fatal("Failed to claim task xyz")
	}

	coord.Done(task)

	if claimed := coord.Claim(task); claimed {
		t.Fatal("Reclaimed task that was marked as done.")
	}
}
<commit_msg>Bump int test sleeps to make Travis happy :(<commit_after>package m_etcd_test

import (
	"errors"
	"testing"
	"time"

	"github.com/coreos/go-etcd/etcd"
	"github.com/lytics/metafora"
	"github.com/lytics/metafora/m_etcd"
	"github.com/lytics/metafora/m_etcd/testutil"
	"github.com/lytics/metafora/statemachine"
)

const recursive = true

// TestSleepTest is an integration test for all of m_etcd's components.
//
func TestSleepTest(t *testing.T) {
	etcdc, hosts := testutil.NewEtcdClient(t)
	t.Parallel()
	const namespace = "sleeptest-metafora"
	const sleepingtasks = "sleeping-task1"

	etcdc.Delete(namespace, recursive)

	holdtask := make(chan bool)
	h := func(task metafora.Task, cmds <-chan *statemachine.Message) *statemachine.Message {

		if task.ID() == sleepingtasks {
			sleeptil := 5 * time.Second
			nextstarttime := (time.Now().Add(sleeptil))
			t.Logf("sleeping task:%v sleepfor:%v", task, nextstarttime)
			<-holdtask
			return statemachine.SleepMessage(nextstarttime)
		}

		cmd := <-cmds
		t.Logf("non sleeping task:%v", task)

		return cmd
	}

	newC := func(name, ns string) *metafora.Consumer {
		conf := m_etcd.NewConfig(ns, hosts)
		conf.Name = name
		coord, hf, bal, err := m_etcd.New(conf, h)
		if err != nil {
			t.Fatalf("Error creating new etcd stack: %v", err)
		}
		cons, err := metafora.NewConsumer(coord, hf, bal)
		if err != nil {
			t.Fatalf("Error creating consumer %s:%s: %v", ns, name, err)
		}
		go func() {
			cons.Run()
			t.Logf("Consumer:%s exited.", name)
		}()
		return cons
	}

	assertRunning := func(tid string, cons ...*metafora.Consumer) {
		found := false
		for _, c := range cons {
			tasks := c.Tasks()
			if len(tasks) > 0 && found {
				t.Fatal("Task already found running but another task is running on a different consumer")
			}
			if len(tasks) > 1 {
				t.Fatalf("Expected at most 1 task, but found: %d", len(tasks))
			}
			if len(tasks) == 1 && tasks[0].Task().ID() == tid {
				found = true
			}
		}
		if !found {
			t.Fatalf("Could not find task=%q", tid)
		}
	}

	// Start 2 consumers
	cons1 := newC("node1", namespace)
	cons2 := newC("node2", namespace)

	// Create clients and start some tests
	cliA := m_etcd.NewClient(namespace, hosts)

	if err := cliA.SubmitTask(m_etcd.DefaultTaskFunc(sleepingtasks, "")); err != nil {
		t.Fatalf("Error submitting task1 to a: %v", err)
	}

	// Give consumers a bit to pick up tasks
	time.Sleep(500 * time.Millisecond)

	assertRunning(sleepingtasks, cons1, cons2)

	holdtask <- true
	// Give consumers a bit to pick up tasks
	time.Sleep(500 * time.Millisecond)

	assertRunning(sleepingtasks, cons1, cons2) // not sure if this should be true or false.

	wait1 := make(chan bool)
	go func() {
		defer close(wait1)
		// Shutdown
		cons1.Shutdown()
		cons2.Shutdown()
	}()

	timeout := time.NewTimer(1 * time.Second)
	select {
	case <-wait1:
	case <-timeout.C:
		t.Fatalf("failed waiting for shutdown")
	}

	//	make sure all tasks are released
	for _, c := range []*metafora.Consumer{cons1, cons2} {
		tasks := c.Tasks()
		for _, work := range tasks {
			t.Fatalf("work id %v is still running", work)
		}
	}
}

// TestAll is an integration test for all of m_etcd's components.
//
// While huge integration tests like this are rarely desirable as they can be
// overly fragile and complex, I found myself manually repeating the tests I've
// automated here over and over. This is far more reliable than expecting
// developers to do adhoc testing of all of the m_etcd package's features.
func TestAll(t *testing.T) {
	etcdc, hosts := testutil.NewEtcdClient(t)
	t.Parallel()

	etcdc.Delete("test-a", recursive)
	etcdc.Delete("test-b", recursive)

	h := func(task metafora.Task, cmds <-chan *statemachine.Message) *statemachine.Message {
		cmd := <-cmds
		if task.ID() == "error-test" {
			return statemachine.ErrorMessage(errors.New("error-test"))
		}
		return cmd
	}

	newC := func(name, ns string) *metafora.Consumer {
		conf := m_etcd.NewConfig(ns, hosts)
		conf.Name = name
		coord, hf, bal, err := m_etcd.New(conf, h)
		if err != nil {
			t.Fatalf("Error creating new etcd stack: %v", err)
		}
		cons, err := metafora.NewConsumer(coord, hf, bal)
		if err != nil {
			t.Fatalf("Error creating consumer %s:%s: %v", ns, name, err)
		}
		go cons.Run()
		return cons
	}
	// Start 4 consumers, 2 per namespace
	cons1a := newC("node1", "test-a")
	cons2a := newC("node2", "test-a")
	cons1b := newC("node1", "test-b")
	cons2b := newC("node2", "test-b")

	// Create clients and start some tests
	cliA := m_etcd.NewClient("test-a", hosts)
	cliB := m_etcd.NewClient("test-b", hosts)

	if err := cliA.SubmitTask(m_etcd.DefaultTaskFunc("task1", "")); err != nil {
		t.Fatalf("Error submitting task1 to a: %v", err)
	}
	if err := cliB.SubmitTask(m_etcd.DefaultTaskFunc("task1", "")); err != nil {
		t.Fatalf("Error submitting task1 to b: %v", err)
	}

	// Give consumers a bit to pick up tasks
	time.Sleep(250 * time.Millisecond)

	assertRunning := func(tid string, cons ...*metafora.Consumer) {
		found := false
		for _, c := range cons {
			tasks := c.Tasks()
			if len(tasks) > 0 && found {
				t.Fatal("Task already found running but another task is running on a different consumer")
			}
			if len(tasks) > 1 {
				t.Fatalf("Expected at most 1 task, but found: %d", len(tasks))
			}
			if len(tasks) == 1 && tasks[0].Task().ID() == tid {
				found = true
			}
		}
		if !found {
			t.Fatalf("Could not find task=%q", tid)
		}
	}

	assertRunning("task1", cons1a, cons2a)
	assertRunning("task1", cons1b, cons2b)

	// Kill task1 in A
	{
		cmdr := m_etcd.NewCommander("test-a", etcdc)
		if err := cmdr.Send("task1", statemachine.KillMessage()); err != nil {
			t.Fatalf("Error sending kill to task1: %v", err)
		}
		time.Sleep(250 * time.Millisecond)

		for _, c := range []*metafora.Consumer{cons1a, cons2a} {
			tasks := c.Tasks()
			if len(tasks) != 0 {
				t.Fatalf("Expected no tasks but found: %d", len(tasks))
			}
		}
	}

	// Submit a bunch of tasks to A
	{
		tasks := []string{"task2", "task3", "task4", "task5", "task6", "task7"}
		for _, tid := range tasks {
			if err := cliA.SubmitTask(m_etcd.DefaultTaskFunc(tid, "")); err != nil {
				t.Fatalf("Error submitting task=%q to A: %v", tid, err)
			}
		}

		// Give them time to start
		time.Sleep(800 * time.Millisecond)

		// Ensure they're balanced
		if err := cliA.SubmitCommand("node1", metafora.CommandBalance()); err != nil {
			t.Fatalf("Error submitting balance command to cons1a: %v", err)
		}
		time.Sleep(800 * time.Millisecond)
		if err := cliA.SubmitCommand("node2", metafora.CommandBalance()); err != nil {
			t.Fatalf("Error submitting balance command to cons1a: %v", err)
		}

		a1tasks := cons1a.Tasks()
		a2tasks := cons2a.Tasks()
		for _, task := range a1tasks {
			metafora.Debug("A1: ", task.Task(), " - ", task.Stopped().IsZero())
		}
		for _, task := range a2tasks {
			metafora.Debug("A2: ", task.Task(), " - ", task.Stopped().IsZero())
		}
		time.Sleep(800 * time.Millisecond)

		a1tasks = cons1a.Tasks()
		a2tasks = cons2a.Tasks()
		if len(a1tasks) < 2 || len(a1tasks) > 4 || len(a2tasks) < 2 || len(a2tasks) > 4 {
			t.Fatalf("Namespace A isn't fairly balanced: node1: %d; node2: %d", len(a1tasks), len(a2tasks))
		}

		// Shutting down a consumer should migrate all tasks to the other
		cons1a.Shutdown()
		time.Sleep(800 * time.Millisecond)

		a2tasks = cons2a.Tasks()
		if len(a2tasks) != len(tasks) {
			t.Fatalf("Consumer 2a should have received all %d tasks but only has %d.", len(tasks), len(a2tasks))
		}
	}

	// Use Namespace B to check Error state handling
	{
		tasks := []string{"task8", "error-test"}
		for _, tid := range tasks {
			if err := cliB.SubmitTask(m_etcd.DefaultTaskFunc(tid, "")); err != nil {
				t.Fatalf("Error submitting task=%q to B: %v", tid, err)
			}
		}

		// Give them time to start
		time.Sleep(time.Second)

		n := len(cons1b.Tasks()) + len(cons2b.Tasks())
		if n != 3 {
			t.Fatalf("Expected B to be running 3 tasks but found %d", n)
		}

		// Resuming error-test 8*2 times should cause it to be failed
		cmdr := m_etcd.NewCommander("test-b", etcdc)
		for i := 0; i < statemachine.DefaultErrMax*2; i++ {
			if err := cmdr.Send("error-test", statemachine.RunMessage()); err != nil {
				t.Fatalf("Unexpected error resuming error-test in B: %v", err)
			}
			time.Sleep(500 * time.Millisecond)
		}

		n = len(cons1b.Tasks()) + len(cons2b.Tasks())
		if n != 2 {
			t.Fatalf("Expected B to be running 2 tasks but found %d", n)
		}

		// Resubmitting a failed task shouldn't error but also shouldn't run.
		if err := cliB.SubmitTask(m_etcd.DefaultTaskFunc("error-test", "")); err != nil {
			t.Fatalf("Error resubmitting error-test task to B: %v", err)
		}

		// Give the statemachine a moment to load the initial state and exit
		time.Sleep(time.Second)

		n = len(cons1b.Tasks()) + len(cons2b.Tasks())
		if n != 2 {
			t.Fatalf("Expected B to be running 2 tasks but found %d", n)
		}
	}

	// Shutdown
	cons2a.Shutdown()
	cons1b.Shutdown()
	cons2b.Shutdown()

	// Make sure everything is cleaned up
	respA, err := etcdc.Get("/test-a/tasks", true, true)
	if err != nil {
		t.Fatalf("Error getting tasks from etcd: %v", err)
	}
	respB, err := etcdc.Get("/test-b/tasks", true, true)
	if err != nil {
		t.Fatalf("Error getting tasks from etcd: %v", err)
	}

	nodes := []*etcd.Node{}
	nodes = append(nodes, respA.Node.Nodes...)
	nodes = append(nodes, respB.Node.Nodes...)
	for _, node := range nodes {
		if len(node.Nodes) > 1 {
			t.Fatalf("%s has %d (>1) nodes. First 2: %s, %s", node.Key, len(node.Nodes), node.Nodes[0].Key, node.Nodes[1].Key)
		}
	}
}

// TestTaskResurrectionInt ensures that a Claim won't recreate a task that had
// been deleted (marked as done). taskmgr has a non-integration version of this
// test.
func TestTaskResurrectionInt(t *testing.T) {
	etcdc, hosts := testutil.NewEtcdClient(t)
	t.Parallel()

	etcdc.Delete("test-resurrect", recursive)

	task := m_etcd.DefaultTaskFunc("xyz", "")

	conf := m_etcd.NewConfig("test-resurrect", hosts)
	coord, err := m_etcd.NewEtcdCoordinator(conf)
	if err != nil {
		t.Fatalf("Error creating coordinator: %v", err)
	}
	if err := coord.Init(nil); err != nil {
		t.Fatalf("Error initializing coordinator: %v", err)
	}

	// Try to claim a nonexistent
	if claimed := coord.Claim(task); claimed {
		t.Fatal("Claiming a nonexistent task should not work but did!")
	}

	// Create a task, mark it as done, and try to claim it again
	client := m_etcd.NewClient("test-resurrect", hosts)
	if err := client.SubmitTask(m_etcd.DefaultTaskFunc("xyz", "")); err != nil {
		t.Fatalf("Error submitting task xyz: %v", err)
	}

	if claimed := coord.Claim(task); !claimed {
		t.Fatal("Failed to claim task xyz")
	}

	coord.Done(task)

	if claimed := coord.Claim(task); claimed {
		t.Fatal("Reclaimed task that was marked as done.")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"

	mp "github.com/mackerelio/go-mackerel-plugin"
	"github.com/mackerelio/mackerel-agent/logging"
)

var logger = logging.GetLogger("metrics.plugin.jvm")

var graphdef map[string](mp.Graphs) = map[string](mp.Graphs){
	"jvm.gc_events": mp.Graphs{
		Label: "JVM GC events",
		Unit:  "integer",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "YGC", Label: "Young GC event", Diff: true},
			mp.Metrics{Name: "FGC", Label: "Full GC event", Diff: true},
		},
	},
	"jvm.gc_time": mp.Graphs{
		Label: "JVM GC time (msec)",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "YGCT", Label: "Young GC time", Diff: true},
			mp.Metrics{Name: "FGCT", Label: "Full GC time", Diff: true},
		},
	},
	"jvm.new_space": mp.Graphs{
		Label: "JVM New Space memory (KB)",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "NGCMX", Label: "New max", Diff: false},
			mp.Metrics{Name: "NGC", Label: "New current", Diff: false},
			mp.Metrics{Name: "EU", Label: "Eden used", Diff: false},
			mp.Metrics{Name: "S0U", Label: "Survivor0 used", Diff: false},
			mp.Metrics{Name: "S1U", Label: "Survivor1 used", Diff: false},
		},
	},
	"jvm.old_space": mp.Graphs{
		Label: "JVM Old Space memory (KB)",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "OGCMX", Label: "Old max", Diff: false},
			mp.Metrics{Name: "OGC", Label: "Old current", Diff: false},
			mp.Metrics{Name: "OU", Label: "Old used", Diff: false},
		},
	},
	"jvm.perm_space": mp.Graphs{
		Label: "JVM Permanent Space (KB)",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "PGCMX", Label: "Perm max", Diff: false},
			mp.Metrics{Name: "PGC", Label: "Perm current", Diff: false},
			mp.Metrics{Name: "PU", Label: "Perm used", Diff: false},
		},
	},
}

type JVMPlugin struct {
	Target    string
	Lvmid     string
	JstatPath string
	JavaName  string
	Tempfile  string
}

// # jps
// 26547 NettyServer
// 6438 Jps
func FetchLvmidByAppname(appname, target, jpsPath string) (string, error) {
	out, err := exec.Command(jpsPath, target).Output()
	if err != nil {
		logger.Errorf("Failed to run exec jps. %s", err)
		return "", err
	}
	for _, line := range strings.Split(string(out), "\n") {
		words := strings.Split(line, " ")
		if len(words) != 2 {
			continue
		}
		lvmid, name := words[0], words[1]
		if name == appname {
			return lvmid, nil
		}
	}
	return "", errors.New(fmt.Sprintf("Cannot get lvmid from %s", appname))
}

func fetchJstatMetrics(lvmid, option, jstatPath string) (map[string]float64, error) {
	out, err := exec.Command(jstatPath, option, lvmid).Output()
	if err != nil {
		logger.Errorf("Failed to run exec jstat. %s", err)
		return nil, err
	}

	lines := strings.Split(string(out), "\n")
	keys := strings.Fields(lines[0])
	values := strings.Fields(lines[1])

	stat := make(map[string]float64)
	for i, key := range keys {
		value, err := strconv.ParseFloat(values[i], 64)
		if err != nil {
			logger.Warningf("Failed to parse value. %s", err)
		}
		stat[key] = value
	}

	return stat, nil
}

func mergeStat(dst, src map[string]float64) {
	for k, v := range src {
		dst[k] = v
	}
}

// # jstat -gc <vmid>
//  S0C    S1C    S0U    S1U      EC       EU        OC         OU       PC     PU    YGC     YGCT    FGC    FGCT     GCT
// 3584.0 3584.0 2528.0  0.0   692224.0 19062.4  1398272.0   485450.1  72704.0 72611.3   3152   30.229   0      0.000   30.229

// # jstat -gccapacity  <vmid>
//  NGCMN    NGCMX     NGC     S0C   S1C       EC      OGCMN      OGCMX       OGC         OC      PGCMN    PGCMX     PGC       PC     YGC    FGC
// 699392.0 699392.0 699392.0 4096.0 4096.0 691200.0  1398272.0  1398272.0  1398272.0  1398272.0  21504.0 524288.0  72704.0  72704.0   4212     0

// # jstat -gcnew  <vmid>
//  S0C    S1C    S0U    S1U   TT MTT  DSS      EC       EU     YGC     YGCT
// 3072.0 3072.0    0.0 2848.0  1  15 3072.0 693248.0 626782.2   3463   33.658

func (m JVMPlugin) FetchMetrics() (map[string]float64, error) {
	gcStat, err := fetchJstatMetrics(m.Lvmid, "-gc", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcCapacityStat, err := fetchJstatMetrics(m.Lvmid, "-gccapacity", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcNewStat, err := fetchJstatMetrics(m.Lvmid, "-gcnew", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcOldStat, err := fetchJstatMetrics(m.Lvmid, "-gcold", m.JstatPath)
	if err != nil {
		return nil, err
	}

	stat := make(map[string]float64)
	mergeStat(stat, gcStat)
	mergeStat(stat, gcCapacityStat)
	mergeStat(stat, gcNewStat)
	mergeStat(stat, gcOldStat)

	return stat, nil
}

func (m JVMPlugin) GraphDefinition() map[string](mp.Graphs) {
	rawJavaName := m.JavaName
	lowerJavaName := strings.ToLower(m.JavaName)
	return map[string](mp.Graphs){
		fmt.Sprintf("jvm.%s.gc_events", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s GC events", rawJavaName),
			Unit:  "integer",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "YGC", Label: "Young GC event", Diff: true},
				mp.Metrics{Name: "FGC", Label: "Full GC event", Diff: true},
			},
		},
		fmt.Sprintf("jvm.%s.gc_time", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s GC time (msec)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "YGCT", Label: "Young GC time", Diff: true},
				mp.Metrics{Name: "FGCT", Label: "Full GC time", Diff: true},
			},
		},
		fmt.Sprintf("jvm.%s.new_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s New Space memory (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "NGCMX", Label: "New max", Diff: false},
				mp.Metrics{Name: "NGC", Label: "New current", Diff: false},
				mp.Metrics{Name: "EU", Label: "Eden used", Diff: false},
				mp.Metrics{Name: "S0U", Label: "Survivor0 used", Diff: false},
				mp.Metrics{Name: "S1U", Label: "Survivor1 used", Diff: false},
			},
		},
		fmt.Sprintf("jvm.%s.old_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s Old Space memory (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "OGCMX", Label: "Old max", Diff: false},
				mp.Metrics{Name: "OGC", Label: "Old current", Diff: false},
				mp.Metrics{Name: "OU", Label: "Old used", Diff: false},
			},
		},
		fmt.Sprintf("jvm.%s.perm_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s Permanent Space (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "PGCMX", Label: "Perm max", Diff: false},
				mp.Metrics{Name: "PGC", Label: "Perm current", Diff: false},
				mp.Metrics{Name: "PU", Label: "Perm used", Diff: false},
			},
		},
	}
}

func main() {
	optHost := flag.String("host", "localhost", "Hostname")
	optPort := flag.String("port", "1099", "Port")
	optJstatPath := flag.String("jstatpath", "/usr/bin/jstat", "jstat path")
	optJpsPath := flag.String("jpspath", "/usr/bin/jps", "jps path")
	optJavaName := flag.String("javaname", "", "Java app name")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	if *optJavaName == "" {
		logger.Errorf("javaname is required")
		flag.PrintDefaults()
		os.Exit(1)
	}

	var jvm JVMPlugin
	jvm.Target = fmt.Sprintf("%s:%s", *optHost, *optPort)
	lvmid, err := FetchLvmidByAppname(*optJavaName, jvm.Target, *optJpsPath)
	if err != nil {
		logger.Errorf("Failed to fetch lvmid. %s", err)
		os.Exit(1)
	}
	jvm.Lvmid = lvmid
	jvm.JstatPath = *optJstatPath
	jvm.JavaName = *optJavaName

	helper := mp.NewMackerelPlugin(jvm)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-jvm-%s", *optHost)
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<commit_msg>Delete grabage<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"

	mp "github.com/mackerelio/go-mackerel-plugin"
	"github.com/mackerelio/mackerel-agent/logging"
)

var logger = logging.GetLogger("metrics.plugin.jvm")

type JVMPlugin struct {
	Target    string
	Lvmid     string
	JstatPath string
	JavaName  string
	Tempfile  string
}

// # jps
// 26547 NettyServer
// 6438 Jps
func FetchLvmidByAppname(appname, target, jpsPath string) (string, error) {
	out, err := exec.Command(jpsPath, target).Output()
	if err != nil {
		logger.Errorf("Failed to run exec jps. %s", err)
		return "", err
	}
	for _, line := range strings.Split(string(out), "\n") {
		words := strings.Split(line, " ")
		if len(words) != 2 {
			continue
		}
		lvmid, name := words[0], words[1]
		if name == appname {
			return lvmid, nil
		}
	}
	return "", errors.New(fmt.Sprintf("Cannot get lvmid from %s", appname))
}

func fetchJstatMetrics(lvmid, option, jstatPath string) (map[string]float64, error) {
	out, err := exec.Command(jstatPath, option, lvmid).Output()
	if err != nil {
		logger.Errorf("Failed to run exec jstat. %s", err)
		return nil, err
	}

	lines := strings.Split(string(out), "\n")
	keys := strings.Fields(lines[0])
	values := strings.Fields(lines[1])

	stat := make(map[string]float64)
	for i, key := range keys {
		value, err := strconv.ParseFloat(values[i], 64)
		if err != nil {
			logger.Warningf("Failed to parse value. %s", err)
		}
		stat[key] = value
	}

	return stat, nil
}

func mergeStat(dst, src map[string]float64) {
	for k, v := range src {
		dst[k] = v
	}
}

// # jstat -gc <vmid>
//  S0C    S1C    S0U    S1U      EC       EU        OC         OU       PC     PU    YGC     YGCT    FGC    FGCT     GCT
// 3584.0 3584.0 2528.0  0.0   692224.0 19062.4  1398272.0   485450.1  72704.0 72611.3   3152   30.229   0      0.000   30.229

// # jstat -gccapacity  <vmid>
//  NGCMN    NGCMX     NGC     S0C   S1C       EC      OGCMN      OGCMX       OGC         OC      PGCMN    PGCMX     PGC       PC     YGC    FGC
// 699392.0 699392.0 699392.0 4096.0 4096.0 691200.0  1398272.0  1398272.0  1398272.0  1398272.0  21504.0 524288.0  72704.0  72704.0   4212     0

// # jstat -gcnew  <vmid>
//  S0C    S1C    S0U    S1U   TT MTT  DSS      EC       EU     YGC     YGCT
// 3072.0 3072.0    0.0 2848.0  1  15 3072.0 693248.0 626782.2   3463   33.658

func (m JVMPlugin) FetchMetrics() (map[string]float64, error) {
	gcStat, err := fetchJstatMetrics(m.Lvmid, "-gc", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcCapacityStat, err := fetchJstatMetrics(m.Lvmid, "-gccapacity", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcNewStat, err := fetchJstatMetrics(m.Lvmid, "-gcnew", m.JstatPath)
	if err != nil {
		return nil, err
	}
	gcOldStat, err := fetchJstatMetrics(m.Lvmid, "-gcold", m.JstatPath)
	if err != nil {
		return nil, err
	}

	stat := make(map[string]float64)
	mergeStat(stat, gcStat)
	mergeStat(stat, gcCapacityStat)
	mergeStat(stat, gcNewStat)
	mergeStat(stat, gcOldStat)

	return stat, nil
}

func (m JVMPlugin) GraphDefinition() map[string](mp.Graphs) {
	rawJavaName := m.JavaName
	lowerJavaName := strings.ToLower(m.JavaName)
	return map[string](mp.Graphs){
		fmt.Sprintf("jvm.%s.gc_events", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s GC events", rawJavaName),
			Unit:  "integer",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "YGC", Label: "Young GC event", Diff: true},
				mp.Metrics{Name: "FGC", Label: "Full GC event", Diff: true},
			},
		},
		fmt.Sprintf("jvm.%s.gc_time", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s GC time (msec)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "YGCT", Label: "Young GC time", Diff: true},
				mp.Metrics{Name: "FGCT", Label: "Full GC time", Diff: true},
			},
		},
		fmt.Sprintf("jvm.%s.new_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s New Space memory (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "NGCMX", Label: "New max", Diff: false},
				mp.Metrics{Name: "NGC", Label: "New current", Diff: false},
				mp.Metrics{Name: "EU", Label: "Eden used", Diff: false},
				mp.Metrics{Name: "S0U", Label: "Survivor0 used", Diff: false},
				mp.Metrics{Name: "S1U", Label: "Survivor1 used", Diff: false},
			},
		},
		fmt.Sprintf("jvm.%s.old_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s Old Space memory (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "OGCMX", Label: "Old max", Diff: false},
				mp.Metrics{Name: "OGC", Label: "Old current", Diff: false},
				mp.Metrics{Name: "OU", Label: "Old used", Diff: false},
			},
		},
		fmt.Sprintf("jvm.%s.perm_space", lowerJavaName): mp.Graphs{
			Label: fmt.Sprintf("JVM %s Permanent Space (KB)", rawJavaName),
			Unit:  "float",
			Metrics: [](mp.Metrics){
				mp.Metrics{Name: "PGCMX", Label: "Perm max", Diff: false},
				mp.Metrics{Name: "PGC", Label: "Perm current", Diff: false},
				mp.Metrics{Name: "PU", Label: "Perm used", Diff: false},
			},
		},
	}
}

func main() {
	optHost := flag.String("host", "localhost", "Hostname")
	optPort := flag.String("port", "1099", "Port")
	optJstatPath := flag.String("jstatpath", "/usr/bin/jstat", "jstat path")
	optJpsPath := flag.String("jpspath", "/usr/bin/jps", "jps path")
	optJavaName := flag.String("javaname", "", "Java app name")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	if *optJavaName == "" {
		logger.Errorf("javaname is required")
		flag.PrintDefaults()
		os.Exit(1)
	}

	var jvm JVMPlugin
	jvm.Target = fmt.Sprintf("%s:%s", *optHost, *optPort)
	lvmid, err := FetchLvmidByAppname(*optJavaName, jvm.Target, *optJpsPath)
	if err != nil {
		logger.Errorf("Failed to fetch lvmid. %s", err)
		os.Exit(1)
	}
	jvm.Lvmid = lvmid
	jvm.JstatPath = *optJstatPath
	jvm.JavaName = *optJavaName

	helper := mp.NewMackerelPlugin(jvm)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-jvm-%s", *optHost)
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<|endoftext|>


<commit_before>package libnetwork

import (
	"encoding/json"
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/libkv/store"
	"github.com/docker/libnetwork/datastore"
	"github.com/docker/libnetwork/types"
)

func (c *controller) validateDatastoreConfig() bool {
	if c.cfg == nil || c.cfg.Datastore.Client.Provider == "" || c.cfg.Datastore.Client.Address == "" {
		return false
	}
	return true
}

func (c *controller) initDataStore() error {
	c.Lock()
	cfg := c.cfg
	c.Unlock()
	if !c.validateDatastoreConfig() {
		return fmt.Errorf("datastore initialization requires a valid configuration")
	}

	store, err := datastore.NewDataStore(&cfg.Datastore)
	if err != nil {
		return err
	}
	c.Lock()
	c.store = store
	c.Unlock()

	nws, err := c.getNetworksFromStore()
	if err == nil {
		c.processNetworkUpdate(nws, nil)
	} else if err != datastore.ErrKeyNotFound {
		log.Warnf("failed to read networks from datastore during init : %v", err)
	}
	return c.watchNetworks()
}

func (c *controller) getNetworksFromStore() ([]*store.KVPair, error) {
	c.Lock()
	cs := c.store
	c.Unlock()
	return cs.KVStore().List(datastore.Key(datastore.NetworkKeyPrefix))
}

func (c *controller) newNetworkFromStore(n *network) error {
	n.Lock()
	n.ctrlr = c
	n.endpoints = endpointTable{}
	n.Unlock()

	return c.addNetwork(n)
}

func (c *controller) updateNetworkToStore(n *network) error {
	global, err := n.isGlobalScoped()
	if err != nil || !global {
		return err
	}
	c.Lock()
	cs := c.store
	c.Unlock()
	if cs == nil {
		log.Debugf("datastore not initialized. Network %s is not added to the store", n.Name())
		return nil
	}

	return cs.PutObjectAtomic(n)
}

func (c *controller) deleteNetworkFromStore(n *network) error {
	global, err := n.isGlobalScoped()
	if err != nil || !global {
		return err
	}
	c.Lock()
	cs := c.store
	c.Unlock()
	if cs == nil {
		log.Debugf("datastore not initialized. Network %s is not deleted from datastore", n.Name())
		return nil
	}

	if err := cs.DeleteObjectAtomic(n); err != nil {
		return err
	}

	return nil
}

func (c *controller) getNetworkFromStore(nid types.UUID) (*network, error) {
	n := network{id: nid}
	if err := c.store.GetObject(datastore.Key(n.Key()...), &n); err != nil {
		return nil, err
	}
	return &n, nil
}

func (c *controller) newEndpointFromStore(key string, ep *endpoint) error {
	ep.Lock()
	n := ep.network
	id := ep.id
	ep.Unlock()

	_, err := n.EndpointByID(string(id))
	if err != nil {
		if _, ok := err.(ErrNoSuchEndpoint); ok {
			return n.addEndpoint(ep)
		}
	}
	return err
}

func (c *controller) updateEndpointToStore(ep *endpoint) error {
	ep.Lock()
	n := ep.network
	name := ep.name
	ep.Unlock()
	global, err := n.isGlobalScoped()
	if err != nil || !global {
		return err
	}
	c.Lock()
	cs := c.store
	c.Unlock()
	if cs == nil {
		log.Debugf("datastore not initialized. endpoint %s is not added to the store", name)
		return nil
	}

	return cs.PutObjectAtomic(ep)
}

func (c *controller) getEndpointFromStore(eid types.UUID) (*endpoint, error) {
	ep := endpoint{id: eid}
	if err := c.store.GetObject(datastore.Key(ep.Key()...), &ep); err != nil {
		return nil, err
	}
	return &ep, nil
}

func (c *controller) deleteEndpointFromStore(ep *endpoint) error {
	ep.Lock()
	n := ep.network
	ep.Unlock()
	global, err := n.isGlobalScoped()
	if err != nil || !global {
		return err
	}

	c.Lock()
	cs := c.store
	c.Unlock()
	if cs == nil {
		log.Debugf("datastore not initialized. endpoint %s is not deleted from datastore", ep.Name())
		return nil
	}

	if err := cs.DeleteObjectAtomic(ep); err != nil {
		return err
	}

	return nil
}

func (c *controller) watchNetworks() error {
	if !c.validateDatastoreConfig() {
		return nil
	}

	c.Lock()
	cs := c.store
	c.Unlock()

	nwPairs, err := cs.KVStore().WatchTree(datastore.Key(datastore.NetworkKeyPrefix), nil)
	if err != nil {
		return err
	}
	go func() {
		for {
			select {
			case nws := <-nwPairs:
				c.Lock()
				tmpview := networkTable{}
				lview := c.networks
				c.Unlock()
				for k, v := range lview {
					global, _ := v.isGlobalScoped()
					if global {
						tmpview[k] = v
					}
				}
				c.processNetworkUpdate(nws, &tmpview)
				// Delete processing
				for k := range tmpview {
					c.Lock()
					existing, ok := c.networks[k]
					c.Unlock()
					if !ok {
						continue
					}
					tmp := network{}
					if err := c.store.GetObject(datastore.Key(existing.Key()...), &tmp); err != datastore.ErrKeyNotFound {
						continue
					}
					if err := existing.deleteNetwork(); err != nil {
						log.Debugf("Delete failed %s: %s", existing.name, err)
					}
				}
			}
		}
	}()
	return nil
}

func (n *network) watchEndpoints() error {
	if !n.ctrlr.validateDatastoreConfig() {
		return nil
	}

	n.Lock()
	cs := n.ctrlr.store
	tmp := endpoint{network: n}
	n.stopWatchCh = make(chan struct{})
	stopCh := n.stopWatchCh
	n.Unlock()

	epPairs, err := cs.KVStore().WatchTree(datastore.Key(tmp.KeyPrefix()...), stopCh)
	if err != nil {
		return err
	}
	go func() {
		for {
			select {
			case <-stopCh:
				return
			case eps := <-epPairs:
				n.Lock()
				tmpview := endpointTable{}
				lview := n.endpoints
				n.Unlock()
				for k, v := range lview {
					global, _ := v.network.isGlobalScoped()
					if global {
						tmpview[k] = v
					}
				}
				for _, epe := range eps {
					var ep endpoint
					err := json.Unmarshal(epe.Value, &ep)
					if err != nil {
						log.Error(err)
						continue
					}
					delete(tmpview, ep.id)
					ep.dbIndex = epe.LastIndex
					ep.network = n
					if n.ctrlr.processEndpointUpdate(&ep) {
						err = n.ctrlr.newEndpointFromStore(epe.Key, &ep)
						if err != nil {
							log.Error(err)
						}
					}
				}
				// Delete processing
				for k := range tmpview {
					n.Lock()
					existing, ok := n.endpoints[k]
					n.Unlock()
					if !ok {
						continue
					}
					tmp := endpoint{}
					if err := cs.GetObject(datastore.Key(existing.Key()...), &tmp); err != datastore.ErrKeyNotFound {
						continue
					}
					if err := existing.deleteEndpoint(); err != nil {
						log.Debugf("Delete failed %s: %s", existing.name, err)
					}
				}
			}
		}
	}()
	return nil
}

func (n *network) stopWatch() {
	n.Lock()
	if n.stopWatchCh != nil {
		close(n.stopWatchCh)
		n.stopWatchCh = nil
	}
	n.Unlock()
}

func (c *controller) processNetworkUpdate(nws []*store.KVPair, prune *networkTable) {
	for _, kve := range nws {
		var n network
		err := json.Unmarshal(kve.Value, &n)
		if err != nil {
			log.Error(err)
			continue
		}
		if prune != nil {
			delete(*prune, n.id)
		}
		n.dbIndex = kve.LastIndex
		c.Lock()
		existing, ok := c.networks[n.id]
		c.Unlock()
		if ok {
			existing.Lock()
			// Skip existing network update
			if existing.dbIndex != n.dbIndex {
				existing.dbIndex = n.dbIndex
				existing.endpointCnt = n.endpointCnt
			}
			existing.Unlock()
			continue
		}

		if err = c.newNetworkFromStore(&n); err != nil {
			log.Error(err)
		}
	}
}

func (c *controller) processEndpointUpdate(ep *endpoint) bool {
	nw := ep.network
	if nw == nil {
		return true
	}
	nw.Lock()
	id := nw.id
	nw.Unlock()

	c.Lock()
	n, ok := c.networks[id]
	c.Unlock()
	if !ok {
		return true
	}
	existing, _ := n.EndpointByID(string(ep.id))
	if existing == nil {
		return true
	}

	ee := existing.(*endpoint)
	ee.Lock()
	if ee.dbIndex != ep.dbIndex {
		ee.dbIndex = ep.dbIndex
		if ee.container != nil && ep.container != nil {
			// we care only about the container id
			ee.container.id = ep.container.id
		} else {
			// we still care only about the container id, but this is a short-cut to communicate join or leave operation
			ee.container = ep.container
		}
	}
	ee.Unlock()

	return false
}
<commit_msg>Remove extra conditional.<commit_after>package libnetwork

import (
	"encoding/json"
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/libkv/store"
	"github.com/docker/libnetwork/datastore"
	"github.com/docker/libnetwork/types"
)

func (c *controller) validateDatastoreConfig() bool {
	return c.cfg != nil && c.cfg.Datastore.Client.Provider != "" && c.cfg.Datastore.Client.Address != ""
}

func (c *controller) initDataStore() error {
	c.Lock()
	cfg := c.cfg
	c.Unlock()
	if !c.validateDatastoreConfig() {
		return fmt.Errorf("datastore initialization requires a valid configuration")
	}

	store, err := datastore.NewDataStore(&cfg.Datastore)
	if err != nil {
		return err
	}
	c.Lock()
	c.store = store
	c.Unlock()

	nws, err := c.getNetworksFromStore()
	if err == nil {
		c.processNetworkUpdate(nws, nil)
	} else if err != datastore.ErrKeyNotFound {
		log.Warnf("failed to read networks from datastore during init : %v", err)
	}
	return c.watchNetworks()
}

func (c *controller) getNetworksFromStore() ([]*store.KVPair, error) {
	c.Lock()
	cs := c.store
	c.Unlock()
	return cs.KVStore().List(datastore.Key(datastore.NetworkKeyPrefix))
}

func (c *controller) newNetworkFromStore(n *network) error {
	n.Lock()
	n.ctrlr = c
	n.endpoints = endpointTable{}
	n.Unlock()

	return c.addNetwork(n)
}

func (c *controller) updateNetworkToStore(n *network) error {
	global, err := n.isGlobalScoped()
	if err != nil || !global {
		return err
	}
	c.Lock()
	cs := c.store
	c.Unlock()
	if cs == nil {
		log.Debugf("datastore not initialized. Network %s is not added to the store", n.Name())
		return nil
	}

	return cs.PutObjectAtomic(n)
}

func (c *controller) deleteNetworkFromStore(n *network) error {
	global, err := n.isGlobalScoped()
	if err != nil || !global {
		return err
	}
	c.Lock()
	cs := c.store
	c.Unlock()
	if cs == nil {
		log.Debugf("datastore not initialized. Network %s is not deleted from datastore", n.Name())
		return nil
	}

	if err := cs.DeleteObjectAtomic(n); err != nil {
		return err
	}

	return nil
}

func (c *controller) getNetworkFromStore(nid types.UUID) (*network, error) {
	n := network{id: nid}
	if err := c.store.GetObject(datastore.Key(n.Key()...), &n); err != nil {
		return nil, err
	}
	return &n, nil
}

func (c *controller) newEndpointFromStore(key string, ep *endpoint) error {
	ep.Lock()
	n := ep.network
	id := ep.id
	ep.Unlock()

	_, err := n.EndpointByID(string(id))
	if err != nil {
		if _, ok := err.(ErrNoSuchEndpoint); ok {
			return n.addEndpoint(ep)
		}
	}
	return err
}

func (c *controller) updateEndpointToStore(ep *endpoint) error {
	ep.Lock()
	n := ep.network
	name := ep.name
	ep.Unlock()
	global, err := n.isGlobalScoped()
	if err != nil || !global {
		return err
	}
	c.Lock()
	cs := c.store
	c.Unlock()
	if cs == nil {
		log.Debugf("datastore not initialized. endpoint %s is not added to the store", name)
		return nil
	}

	return cs.PutObjectAtomic(ep)
}

func (c *controller) getEndpointFromStore(eid types.UUID) (*endpoint, error) {
	ep := endpoint{id: eid}
	if err := c.store.GetObject(datastore.Key(ep.Key()...), &ep); err != nil {
		return nil, err
	}
	return &ep, nil
}

func (c *controller) deleteEndpointFromStore(ep *endpoint) error {
	ep.Lock()
	n := ep.network
	ep.Unlock()
	global, err := n.isGlobalScoped()
	if err != nil || !global {
		return err
	}

	c.Lock()
	cs := c.store
	c.Unlock()
	if cs == nil {
		log.Debugf("datastore not initialized. endpoint %s is not deleted from datastore", ep.Name())
		return nil
	}

	if err := cs.DeleteObjectAtomic(ep); err != nil {
		return err
	}

	return nil
}

func (c *controller) watchNetworks() error {
	if !c.validateDatastoreConfig() {
		return nil
	}

	c.Lock()
	cs := c.store
	c.Unlock()

	nwPairs, err := cs.KVStore().WatchTree(datastore.Key(datastore.NetworkKeyPrefix), nil)
	if err != nil {
		return err
	}
	go func() {
		for {
			select {
			case nws := <-nwPairs:
				c.Lock()
				tmpview := networkTable{}
				lview := c.networks
				c.Unlock()
				for k, v := range lview {
					global, _ := v.isGlobalScoped()
					if global {
						tmpview[k] = v
					}
				}
				c.processNetworkUpdate(nws, &tmpview)
				// Delete processing
				for k := range tmpview {
					c.Lock()
					existing, ok := c.networks[k]
					c.Unlock()
					if !ok {
						continue
					}
					tmp := network{}
					if err := c.store.GetObject(datastore.Key(existing.Key()...), &tmp); err != datastore.ErrKeyNotFound {
						continue
					}
					if err := existing.deleteNetwork(); err != nil {
						log.Debugf("Delete failed %s: %s", existing.name, err)
					}
				}
			}
		}
	}()
	return nil
}

func (n *network) watchEndpoints() error {
	if !n.ctrlr.validateDatastoreConfig() {
		return nil
	}

	n.Lock()
	cs := n.ctrlr.store
	tmp := endpoint{network: n}
	n.stopWatchCh = make(chan struct{})
	stopCh := n.stopWatchCh
	n.Unlock()

	epPairs, err := cs.KVStore().WatchTree(datastore.Key(tmp.KeyPrefix()...), stopCh)
	if err != nil {
		return err
	}
	go func() {
		for {
			select {
			case <-stopCh:
				return
			case eps := <-epPairs:
				n.Lock()
				tmpview := endpointTable{}
				lview := n.endpoints
				n.Unlock()
				for k, v := range lview {
					global, _ := v.network.isGlobalScoped()
					if global {
						tmpview[k] = v
					}
				}
				for _, epe := range eps {
					var ep endpoint
					err := json.Unmarshal(epe.Value, &ep)
					if err != nil {
						log.Error(err)
						continue
					}
					delete(tmpview, ep.id)
					ep.dbIndex = epe.LastIndex
					ep.network = n
					if n.ctrlr.processEndpointUpdate(&ep) {
						err = n.ctrlr.newEndpointFromStore(epe.Key, &ep)
						if err != nil {
							log.Error(err)
						}
					}
				}
				// Delete processing
				for k := range tmpview {
					n.Lock()
					existing, ok := n.endpoints[k]
					n.Unlock()
					if !ok {
						continue
					}
					tmp := endpoint{}
					if err := cs.GetObject(datastore.Key(existing.Key()...), &tmp); err != datastore.ErrKeyNotFound {
						continue
					}
					if err := existing.deleteEndpoint(); err != nil {
						log.Debugf("Delete failed %s: %s", existing.name, err)
					}
				}
			}
		}
	}()
	return nil
}

func (n *network) stopWatch() {
	n.Lock()
	if n.stopWatchCh != nil {
		close(n.stopWatchCh)
		n.stopWatchCh = nil
	}
	n.Unlock()
}

func (c *controller) processNetworkUpdate(nws []*store.KVPair, prune *networkTable) {
	for _, kve := range nws {
		var n network
		err := json.Unmarshal(kve.Value, &n)
		if err != nil {
			log.Error(err)
			continue
		}
		if prune != nil {
			delete(*prune, n.id)
		}
		n.dbIndex = kve.LastIndex
		c.Lock()
		existing, ok := c.networks[n.id]
		c.Unlock()
		if ok {
			existing.Lock()
			// Skip existing network update
			if existing.dbIndex != n.dbIndex {
				existing.dbIndex = n.dbIndex
				existing.endpointCnt = n.endpointCnt
			}
			existing.Unlock()
			continue
		}

		if err = c.newNetworkFromStore(&n); err != nil {
			log.Error(err)
		}
	}
}

func (c *controller) processEndpointUpdate(ep *endpoint) bool {
	nw := ep.network
	if nw == nil {
		return true
	}
	nw.Lock()
	id := nw.id
	nw.Unlock()

	c.Lock()
	n, ok := c.networks[id]
	c.Unlock()
	if !ok {
		return true
	}
	existing, _ := n.EndpointByID(string(ep.id))
	if existing == nil {
		return true
	}

	ee := existing.(*endpoint)
	ee.Lock()
	if ee.dbIndex != ep.dbIndex {
		ee.dbIndex = ep.dbIndex
		if ee.container != nil && ep.container != nil {
			// we care only about the container id
			ee.container.id = ep.container.id
		} else {
			// we still care only about the container id, but this is a short-cut to communicate join or leave operation
			ee.container = ep.container
		}
	}
	ee.Unlock()

	return false
}
<|endoftext|>


<commit_before>// Copyright 2016 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

// Package modules 模块的依赖工具，可以保证各个模块能按顺序初始化依赖项。
package modules

import (
	"errors"
	"fmt"
	"sync"
)

var (
	// ErrModuleExists 表示指定的模块已经存在。
	ErrModuleExists = errors.New("该模块已经存在")

	// ErrModulesInited 表示模块已经初始化完成，不能再添加新模发起人。
	ErrModulesInited = errors.New("所有模块已经初始化完成")
)

// Init 模块的初始化函数，实现者需要注意：
// 在重启服务时，会被再次调用，需要保证在多次调用的情况下，
// 不会出现冲突。
type Init func() error

// 用以表示一个模块所需要的数据。
type module struct {
	name   string   // 模块名
	init   Init     // 初始化函数
	inited bool     // 是否已经初始化
	deps   []string // 依赖项
}

// Modules 模块管理工具，管理模块的初始化顺序
type Modules struct {
	modules map[string]*module
	lock    sync.Mutex
	inited  bool // 是否已经初始化
}

// New 声明一个 Modules 实例
func New() *Modules {
	return &Modules{
		modules: make(map[string]*module, 20),
	}
}

// New 注册一个新的模块，以及它的依赖的模块。
//
// name 为模块的名称；
// init 为模块的初始化函数；
// deps 为模块的依赖模块，依赖模块可以后于当前模块注册。
func (ms *Modules) New(name string, init Init, deps ...string) error {
	if ms.inited {
		return ErrModulesInited
	}

	ms.lock.Lock()
	defer ms.lock.Unlock()

	if _, found := ms.modules[name]; found {
		return ErrModuleExists
	}

	ms.modules[name] = &module{
		name: name,
		init: init,
		deps: deps,
	}
	return nil
}

// Init 对所有的模块进行初始化操作，会进行依赖检测。
// 若模块初始化出错，则会中断并返回出错信息。
func (ms *Modules) Init() error {
	if ms.inited { // 已全部初始化，则不再进行相应操作
		return nil
	}

	ms.lock.Lock()
	defer ms.lock.Unlock()

	for _, m := range ms.modules {
		if err := ms.checkDeps(m); err != nil {
			return err
		}
	}

	for _, m := range ms.modules {
		if err := ms.init(m); err != nil {
			return err
		}
	}

	ms.inited = true
	return nil
}

// Reset 重置为未初始化状态。
func (ms *Modules) Reset() error {
	ms.inited = false

	// 重置所有模块为未初始化状态
	ms.lock.Lock()
	for _, m := range ms.modules {
		m.inited = false
	}
	ms.lock.Unlock()

	return ms.Init()
}

// 初始化指定模块，会先初始化其依赖模块。
// 若该模块已经初始化，则不会作任何操作，包括依赖模块的初始化，也不会执行。
func (ms *Modules) init(m *module) error {
	if m.inited {
		return nil
	}

	// 先初始化依赖项
	for _, dep := range m.deps {
		depm, found := ms.modules[dep]
		if !found {
			return fmt.Errorf("依赖项[%v]未找到", dep)
		}

		if err := ms.init(depm); err != nil {
			return err
		}
	}

	// 初始化模块自身
	if err := m.init(); err != nil {
		return err
	}

	m.inited = true
	return nil
}

// 检测模块的依赖关系。比如：
// 依赖项是否存在；是否存在自我依赖等。
func (ms *Modules) checkDeps(m *module) error {
	// 检测依赖项是否都存在
	for _, dep := range m.deps {
		_, found := ms.modules[dep]
		if !found {
			return fmt.Errorf("未找到[%v]的依赖模块[%v]", m.name, dep)
		}
	}

	if ms.isDep(m.name, m.name) {
		return fmt.Errorf("存在循环依赖项:[%v]", m.name)
	}

	return nil
}

// m1 是否依赖 m2
func (ms *Modules) isDep(m1, m2 string) bool {
	module1, found := ms.modules[m1]
	if !found {
		return false
	}

	for _, dep := range module1.deps {
		if dep == m2 {
			return true
		}

		if _, found = ms.modules[dep]; found {
			if ms.isDep(dep, m2) {
				return true
			}
		}
	}

	return false
}
<commit_msg>更详细的错误提示<commit_after>// Copyright 2016 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

// Package modules 模块的依赖工具，可以保证各个模块能按顺序初始化依赖项。
package modules

import (
	"errors"
	"fmt"
	"sync"
)

// ErrModulesInited 表示模块已经初始化完成，不能再添加新模块。
var ErrModulesInited = errors.New("所有模块已经初始化完成")

// Init 模块的初始化函数，实现者需要注意：
// 在重启服务时，会被再次调用，需要保证在多次调用的情况下，
// 不会出现冲突。
type Init func() error

// 用以表示一个模块所需要的数据。
type module struct {
	name   string   // 模块名
	init   Init     // 初始化函数
	inited bool     // 是否已经初始化
	deps   []string // 依赖项
}

// Modules 模块管理工具，管理模块的初始化顺序
type Modules struct {
	modules map[string]*module
	lock    sync.Mutex
	inited  bool // 是否已经初始化
}

// New 声明一个 Modules 实例
func New() *Modules {
	return &Modules{
		modules: make(map[string]*module, 20),
	}
}

// New 注册一个新的模块，以及它的依赖的模块。
//
// name 为模块的名称；
// init 为模块的初始化函数；
// deps 为模块的依赖模块，依赖模块可以后于当前模块注册。
func (ms *Modules) New(name string, init Init, deps ...string) error {
	if ms.inited {
		return ErrModulesInited
	}

	ms.lock.Lock()
	defer ms.lock.Unlock()

	if _, found := ms.modules[name]; found {
		return fmt.Errorf("模块[%v]已经存在", name)
	}

	ms.modules[name] = &module{
		name: name,
		init: init,
		deps: deps,
	}
	return nil
}

// Init 对所有的模块进行初始化操作，会进行依赖检测。
// 若模块初始化出错，则会中断并返回出错信息。
func (ms *Modules) Init() error {
	if ms.inited { // 已全部初始化，则不再进行相应操作
		return nil
	}

	ms.lock.Lock()
	defer ms.lock.Unlock()

	for _, m := range ms.modules {
		if err := ms.checkDeps(m); err != nil {
			return err
		}
	}

	for _, m := range ms.modules {
		if err := ms.init(m); err != nil {
			return err
		}
	}

	ms.inited = true
	return nil
}

// Reset 重置为未初始化状态。
func (ms *Modules) Reset() error {
	ms.inited = false

	// 重置所有模块为未初始化状态
	ms.lock.Lock()
	for _, m := range ms.modules {
		m.inited = false
	}
	ms.lock.Unlock()

	return ms.Init()
}

// 初始化指定模块，会先初始化其依赖模块。
// 若该模块已经初始化，则不会作任何操作，包括依赖模块的初始化，也不会执行。
func (ms *Modules) init(m *module) error {
	if m.inited {
		return nil
	}

	// 先初始化依赖项
	for _, dep := range m.deps {
		depm, found := ms.modules[dep]
		if !found {
			return fmt.Errorf("依赖项[%v]未找到", dep)
		}

		if err := ms.init(depm); err != nil {
			return err
		}
	}

	// 初始化模块自身
	if err := m.init(); err != nil {
		return err
	}

	m.inited = true
	return nil
}

// 检测模块的依赖关系。比如：
// 依赖项是否存在；是否存在自我依赖等。
func (ms *Modules) checkDeps(m *module) error {
	// 检测依赖项是否都存在
	for _, dep := range m.deps {
		_, found := ms.modules[dep]
		if !found {
			return fmt.Errorf("未找到[%v]的依赖模块[%v]", m.name, dep)
		}
	}

	if ms.isDep(m.name, m.name) {
		return fmt.Errorf("存在循环依赖项:[%v]", m.name)
	}

	return nil
}

// m1 是否依赖 m2
func (ms *Modules) isDep(m1, m2 string) bool {
	module1, found := ms.modules[m1]
	if !found {
		return false
	}

	for _, dep := range module1.deps {
		if dep == m2 {
			return true
		}

		if _, found = ms.modules[dep]; found {
			if ms.isDep(dep, m2) {
				return true
			}
		}
	}

	return false
}
<|endoftext|>


<commit_before>package main

import (
//	"net"
	"net/http"
	"log"
	"flag"
	"os"
	"path/filepath"
	"regexp"
	"fmt"
//	"reflect"
	"strconv"
	"time"
//	"strings"
	"html/template"
	"encoding/json"
	"runtime/pprof"
	"code.google.com/p/go.net/websocket"
	"io/ioutil"
	"os/signal"
	"syscall"
)

var (
	rootdir string
	port string
	address string
	verbose bool
	database string
)

var (
	d Query
	h Hub
)

var templates *template.Template
var homeTemplate *template.Template

func init() {
	flag.StringVar(&rootdir, "root", "current directory", "webroot directory")
	flag.StringVar(&rootdir, "d", "current directory", "webroot directory" + " (shorthand)")
	flag.StringVar(&port,"port", "8080", "listen port")
	flag.StringVar(&port,"p", "8080", "listen port" + " (shorthand)")
	flag.StringVar(&address, "address", "*", "listen address")
	flag.StringVar(&address, "l", "*", "listen address" + " (shorthand)")
	flag.BoolVar(&verbose, "verbose", false, "be verbose")
	flag.BoolVar(&verbose, "v", false, "be verbose" + " (shorthand)")
	flag.StringVar(&database, "database", "db.json", "database file")
}

func debug(s string) {
	if verbose {
		log.Print(s)
	}
}

func debugln(v ...interface{}) {
	if verbose {
		log.Println(v ...)
	}
}

var validLog = regexp.MustCompile("^/log/([a-zA-Z0-9-]+)/(-?[0-9]+[.]{0,1}[0-9]*)(/([ts])/([0-9]+))?/?$")
var validQuery = regexp.MustCompile("^/q/([a-zA-Z0-9-]+)/?$")
var validURLs = regexp.MustCompile("^/")

func logHandler(w http.ResponseWriter, r *http.Request) {
	m := validLog.FindStringSubmatch(r.URL.Path)
	if len(m) == 0 {
		http.Error(w, "Sensor not found", http.StatusNotFound)
		return
	}
	debug("Sensor " + m[1] + " sent value " + m[2])
	if m[4] != "" {
		t, _ := strconv.ParseInt(m[5], 10, 64)
		var tnew time.Time
		if m[4] == "t" {
			tnew = time.Unix(t, 0)
		} else { // m[4] == "s"
			tnew = time.Unix(time.Now().Unix() - t, 0)
		}
		_, told := d.Last(m[1])

		if tnew.After(told) {
			d.StoreT(m[1], m[2], tnew)
		} else {
			http.Error(w, "Sensor send out of order timestamp", http.StatusNotFound)
			h.Pipe <- HometickerJson{"Out of Order Reading", "fa-times-circle", "danger", "Sensor " + m[1] + " out of order value " + m[2] + ". Ignored."}
			return
		}
	} else {
		if ! d.Exists(m[1]) { // Remove when you add code to add/delete sensors instead of adding them automatically.
			h.Pipe <- HometickerJson{"New Sensor", "fa-check-circle", "success", "Sensor " + m[1] + " succesfully added."}
		}
		d.Store(m[1], m[2])
		h.Pipe <- HometickerJson{"New Reading", "fa-plus-circle", "info", "Sensor " + m[1] + " sent value " + m[2] + "."}
	}
//	debugln("Sensor " + m[1] + " now contains:", d.Load(m[1]))
	fmt.Fprintf(w, "ok")
}

func queryHandler(w http.ResponseWriter, r *http.Request) {
	m := validQuery.FindStringSubmatch(r.URL.Path)
        if len(m) == 0 {
                http.Error(w, "Sensor not found", http.StatusNotFound)
                return
        }
	if ! d.Exists(m[1]) {
		http.Error(w, "Sensor not found", http.StatusNotFound)
                return
        }
	debugln("Query for sensor " + m[1])
	a, _ := json.Marshal(d.Load(m[1]))
	fmt.Fprintf(w, string(a))
}

func reloadHandler(w http.ResponseWriter, r *http.Request) {
	loadTemplates()
	fmt.Fprintf(w, "templates reloaded")
}

type HomePage struct {
	Title string
	SensorList template.HTML
	CustomScript template.HTML
}

const homeCustomScript = template.HTML(`<script src="/assets/cjs/hometicker.js"></script>`)

func homeHandler(w http.ResponseWriter, r *http.Request) {
	var sl string
	for _, s := range d.List() {
		sl += `

                   <a href="view/` + s + `">` + s + `

`
	}
	err := templates.ExecuteTemplate(w,
		"home.html",
		HomePage{"Datanomics alpha", template.HTML(sl),	homeCustomScript})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func makeHandler(fn func (http.ResponseWriter, *http.Request)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		m := validURLs.FindStringSubmatch(r.URL.Path)
		if m == nil {
			http.NotFound(w, r)
			return
		}
		fn(w, r)
	}
}

func loadTemplates() {
	templates = template.Must(template.ParseFiles(rootdir + "/templates/header.html",
		rootdir + "/templates/menu.html",
		rootdir + "/templates/footer.html",
		rootdir + "/templates/home.html"))
}

var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")

func main() {
	flag.Parse()
	var err error
	if rootdir == "current directory" {
		rootdir, err =  filepath.Abs(filepath.Dir(os.Args[0]))
	} else {
		rootdir, err = filepath.Abs(filepath.Dir(rootdir))
	}
	if err != nil {
		log.Fatal(err)
	}

	if address == "*" {
		address = ""
	}

	// CPU profiling
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	t := Database{ make(map[string] sensorlog) }
	file, err := ioutil.ReadFile(database)
	if err != nil {
		log.Println("Using new database.")
	} else if err = json.Unmarshal(file, &t); err != nil {
		log.Println("Database corrupt. Creating new.", err)
	} else {
		log.Println("Loaded database.")
	}

	d = &t
	loadTemplates()

	h.Connections = make(map[*Socket]bool)
	h.Pipe = make(chan HometickerJson, 1)
	go h.Broadcast()

	go cleanup()

	http.HandleFunc("/log/", logHandler)
	http.HandleFunc("/q/", queryHandler)
	http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir(rootdir + "/assets"))))
	http.HandleFunc("/reload/", reloadHandler)
	http.Handle("/_hometicker", websocket.Handler(homeTickerHandler))
	http.HandleFunc("/", makeHandler(homeHandler))

	log.Print("Starting webserver. Listening on " + address + ":" + port)
	log.Print("Webroot set to \"" + rootdir + "\".")
	err = http.ListenAndServe(address + ":" + port, nil)
	if err != nil {
		log.Fatal("Couldn't start server. ListenAndServe: ", err)
	}
}

func cleanup() {
        ch := make(chan os.Signal)
        signal.Notify(ch, syscall.SIGINT)
        <-ch
	log.Println("Writing database to disk.")
	dbs, _ := json.Marshal(d)
	err := ioutil.WriteFile(database, dbs, 0600)
	if err != nil {
		log.Println("Error saving database.")
	}
	log.Println("Exiting. Goodbye.")
	os.Exit(1)
}
<commit_msg>Added support for sensor pages. Needed to refactor a bit and make some classes more generic. We pass the regexp to makeHandler() now. Also made some URLs absolute to work around issues.<commit_after>package main

import (
//	"net"
	"net/http"
	"log"
	"flag"
	"os"
	"path/filepath"
	"regexp"
	"fmt"
//	"reflect"
	"strconv"
	"time"
//	"strings"
	"html/template"
	"encoding/json"
	"runtime/pprof"
	"code.google.com/p/go.net/websocket"
	"io/ioutil"
	"os/signal"
	"syscall"
)

var (
	rootdir string
	port string
	address string
	verbose bool
	database string
)

var (
	d Query
	h Hub
)

var templates *template.Template
var homeTemplate *template.Template

func init() {
	flag.StringVar(&rootdir, "root", "current directory", "webroot directory")
	flag.StringVar(&rootdir, "d", "current directory", "webroot directory" + " (shorthand)")
	flag.StringVar(&port,"port", "8080", "listen port")
	flag.StringVar(&port,"p", "8080", "listen port" + " (shorthand)")
	flag.StringVar(&address, "address", "*", "listen address")
	flag.StringVar(&address, "l", "*", "listen address" + " (shorthand)")
	flag.BoolVar(&verbose, "verbose", false, "be verbose")
	flag.BoolVar(&verbose, "v", false, "be verbose" + " (shorthand)")
	flag.StringVar(&database, "database", "db.json", "database file")
}

func debug(s string) {
	if verbose {
		log.Print(s)
	}
}

func debugln(v ...interface{}) {
	if verbose {
		log.Println(v ...)
	}
}

var validLog = regexp.MustCompile("^/log/([a-zA-Z0-9-]+)/(-?[0-9]+[.]{0,1}[0-9]*)(/([ts])/([0-9]+))?/?$")
var validQuery = regexp.MustCompile("^/q/([a-zA-Z0-9-]+)/?$")
var validRoot = regexp.MustCompile("^/$")
var validView = regexp.MustCompile("^/view/([a-zA-Z0-9-]+)/?$")

func logHandler(w http.ResponseWriter, r *http.Request) {
	m := validLog.FindStringSubmatch(r.URL.Path)
	if len(m) == 0 {
		http.Error(w, "Sensor not found", http.StatusNotFound)
		return
	}
	debug("Sensor " + m[1] + " sent value " + m[2])
	if m[4] != "" {
		t, _ := strconv.ParseInt(m[5], 10, 64)
		var tnew time.Time
		if m[4] == "t" {
			tnew = time.Unix(t, 0)
		} else { // m[4] == "s"
			tnew = time.Unix(time.Now().Unix() - t, 0)
		}
		_, told := d.Last(m[1])

		if tnew.After(told) {
			d.StoreT(m[1], m[2], tnew)
		} else {
			http.Error(w, "Sensor send out of order timestamp", http.StatusNotFound)
			h.Pipe <- HometickerJson{"Out of Order Reading", "fa-times-circle", "danger", "Sensor " + m[1] + " out of order value " + m[2] + ". Ignored."}
			return
		}
	} else {
		if ! d.Exists(m[1]) { // Remove when you add code to add/delete sensors instead of adding them automatically.
			h.Pipe <- HometickerJson{"New Sensor", "fa-check-circle", "success", "Sensor " + m[1] + " succesfully added."}
		}
		d.Store(m[1], m[2])
		h.Pipe <- HometickerJson{"New Reading", "fa-plus-circle", "info", "Sensor " + m[1] + " sent value " + m[2] + "."}
	}
//	debugln("Sensor " + m[1] + " now contains:", d.Load(m[1]))
	fmt.Fprintf(w, "ok")
}

func queryHandler(w http.ResponseWriter, r *http.Request) {
	m := validQuery.FindStringSubmatch(r.URL.Path)
        if len(m) == 0 {
                http.Error(w, "Sensor not found", http.StatusNotFound)
                return
        }
	if ! d.Exists(m[1]) {
		http.Error(w, "Sensor not found", http.StatusNotFound)
                return
        }
	debugln("Query for sensor " + m[1])
	a, _ := json.Marshal(d.Load(m[1]))
	fmt.Fprintf(w, string(a))
}

func reloadHandler(w http.ResponseWriter, r *http.Request) {
	loadTemplates()
	fmt.Fprintf(w, "templates reloaded")
}

func sensorList() template.HTML { // When we add/remove sensors manually, make this run once and store its value for performance?
	var sl string
	for _, s := range d.List() {
		sl += `

                   <a href="/view/` + s + `">` + s + `

`
	}
	return template.HTML(sl)
}

type HomePage struct {
	Title string
	SensorList template.HTML
	CustomScript template.HTML
}

const homeCustomScript = template.HTML(`<script src="/assets/cjs/hometicker.js"></script>`)

func homeHandler(w http.ResponseWriter, r *http.Request) {
	err := templates.ExecuteTemplate(w,
		"home.html",
		HomePage{"Datanomics alpha", sensorList(), homeCustomScript})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

type ViewPage struct {
	Title string
	Sensor string
	Content string
	SensorList template.HTML
	CustomScript template.HTML
}
const viewCustomScript = template.HTML("")

func viewHandler(w http.ResponseWriter, r *http.Request) {
	m := validView.FindStringSubmatch(r.URL.Path)
        if ! d.Exists(m[1]) {
		err := templates.ExecuteTemplate(w,
			"sensor.html",
			ViewPage{"Datanomics alpha | Sensor not found", "Error", "Sensor not found", sensorList(), viewCustomScript})
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	} else {
		err := templates.ExecuteTemplate(w,
                        "sensor.html",
                        ViewPage{"Datanomics alpha | " + m[1], m[1], "To be implemented", sensorList(), viewCustomScript})
                if err != nil {
                        http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	}

}

func makeHandler(fn func (http.ResponseWriter, *http.Request), rexp regexp.Regexp) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		m := rexp.FindStringSubmatch(r.URL.Path)
		if m == nil {
			http.NotFound(w, r)
			return
		}
		fn(w, r)
	}
}

func loadTemplates() {
	templates = template.Must(template.ParseFiles(rootdir + "/templates/header.html",
		rootdir + "/templates/menu.html",
		rootdir + "/templates/footer.html",
		rootdir + "/templates/home.html",
		rootdir + "/templates/sensor.html"))
}

var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")

func main() {
	flag.Parse()
	var err error
	if rootdir == "current directory" {
		rootdir, err =  filepath.Abs(filepath.Dir(os.Args[0]))
	} else {
		rootdir, err = filepath.Abs(filepath.Dir(rootdir))
	}
	if err != nil {
		log.Fatal(err)
	}

	if address == "*" {
		address = ""
	}

	// CPU profiling
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}
	t := Database{ make(map[string] sensorlog) }
	file, err := ioutil.ReadFile(database)
	if err != nil {
		log.Println("Using new database.")
	} else if err = json.Unmarshal(file, &t); err != nil {
		log.Println("Database corrupt. Creating new.", err)
	} else {
		log.Println("Loaded database.")
	}

	d = &t
	loadTemplates()

	h.Connections = make(map[*Socket]bool)
	h.Pipe = make(chan HometickerJson, 1)
	go h.Broadcast()

	go cleanup()

	http.HandleFunc("/log/", logHandler)
	http.HandleFunc("/q/", queryHandler)
	http.Handle("/assets/", http.StripPrefix("/assets/", http.FileServer(http.Dir(rootdir + "/assets"))))
	http.HandleFunc("/reload/", reloadHandler)
	http.Handle("/_hometicker", websocket.Handler(homeTickerHandler))
	http.HandleFunc("/view/", makeHandler(viewHandler, *validView))
	http.HandleFunc("/", makeHandler(homeHandler, *validRoot))

	log.Print("Starting webserver. Listening on " + address + ":" + port)
	log.Print("Webroot set to \"" + rootdir + "\".")
	err = http.ListenAndServe(address + ":" + port, nil)
	if err != nil {
		log.Fatal("Couldn't start server. ListenAndServe: ", err)
	}
}

func cleanup() {
        ch := make(chan os.Signal)
        signal.Notify(ch, syscall.SIGINT)
        <-ch
	log.Println("Writing database to disk.")
	dbs, _ := json.Marshal(d)
	err := ioutil.WriteFile(database, dbs, 0600)
	if err != nil {
		log.Println("Error saving database.")
	}
	log.Println("Exiting. Goodbye.")
	os.Exit(1)
}
<|endoftext|>


<commit_before>package clicommand

import (
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/buildkite/agent/v3/api"
	"github.com/buildkite/agent/v3/cliconfig"
	"github.com/buildkite/roko"
	"github.com/urfave/cli"
)

type OidcTokenConfig struct {
	Audience string `cli:"audience"`
	Job      string `cli:"job"      validate:"required"`

	// Global flags
	Debug       bool     `cli:"debug"`
	LogLevel    string   `cli:"log-level"`
	NoColor     bool     `cli:"no-color"`
	Experiments []string `cli:"experiment" normalize:"list"`
	Profile     string   `cli:"profile"`

	// API config
	DebugHTTP        bool   `cli:"debug-http"`
	AgentAccessToken string `cli:"agent-access-token" validate:"required"`
	Endpoint         string `cli:"endpoint"           validate:"required"`
	NoHTTP2          bool   `cli:"no-http2"`
}

const (
	oidcTokenDescription = `Usage:

   buildkite-agent oidc token [options...]

Description:
   Requests and prints an OIDC token from Buildkite that claims the Job ID and
	 the specified audience. If no audience is specified, the endpoint's default
	 audience will be claimed.

Example:
   $ buildkite-agent oidc token --audience sts.amazonaws.com

   Prints the environment passed into the process
`
	backoffSeconds = 2
	maxAttempts    = 5
)

var OidcTokenCommand = cli.Command{
	Name:        "token",
	Usage:       "Requests and prints an OIDC token from Buildkite with the specified audience,",
	Description: oidcTokenDescription,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "audience",
			Value: "",
			Usage: "The audience that will consume the OIDC token",
		},
		cli.StringFlag{
			Name:   "job",
			Value:  "",
			Usage:  "Buildkite Job Id to claim in the OIDC token",
			EnvVar: "BUILDKITE_JOB_ID",
		},

		// API Flags
		AgentAccessTokenFlag,
		EndpointFlag,
		NoHTTP2Flag,
		DebugHTTPFlag,

		// Global flags
		NoColorFlag,
		DebugFlag,
		LogLevelFlag,
		ExperimentsFlag,
		ProfileFlag,
	},
	Action: func(c *cli.Context) error {
		// The configuration will be loaded into this struct
		cfg := OidcTokenConfig{}

		loader := cliconfig.Loader{CLI: c, Config: &cfg}
		warnings, err := loader.Load()
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err)
			os.Exit(1)
		}

		l := CreateLogger(&cfg)

		// Now that we have a logger, log out the warnings that loading config generated
		for _, warning := range warnings {
			l.Warn("%s", warning)
		}

		// Setup any global configuration options
		done := HandleGlobalFlags(l, cfg)
		defer done()

		// Create the API client
		client := api.NewClient(l, loadAPIClientConfig(cfg, "AgentAccessToken"))

		// Find the meta data value
		var token *api.OidcToken
		var resp *api.Response

		if err := roko.NewRetrier(
			roko.WithMaxAttempts(maxAttempts),
			roko.WithStrategy(roko.Exponential(backoffSeconds*time.Second, 0)),
		).Do(func(r *roko.Retrier) error {
			var audience []string
			if len(cfg.Audience) > 0 {
				audience = []string{cfg.Audience}
			}

			token, resp, err = client.OidcToken(cfg.Job, audience...)
			if resp != nil {
				fmt.Println(resp.StatusCode, r.ShouldGiveUp())

				switch resp.StatusCode {
				// Don't bother retrying if the response was one of these statuses
				case http.StatusUnauthorized, http.StatusForbidden, http.StatusUnprocessableEntity:
					r.Break()
					return err
				}
			}

			if err != nil {
				l.Warn("%s (%s)", err, r)
			}

			return err
		}); err != nil {
			if len(cfg.Audience) > 0 {
				l.Error("Could not obtain OIDC token for audience %s", cfg.Audience)
			} else {
				l.Error("Could not obtain OIDC token for default audience")
			}
			return err
		}

		fmt.Println(token.Token)
		return nil
	},
}
<commit_msg>Add skipping of retry for 400 errors<commit_after>package clicommand

import (
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/buildkite/agent/v3/api"
	"github.com/buildkite/agent/v3/cliconfig"
	"github.com/buildkite/roko"
	"github.com/urfave/cli"
)

type OidcTokenConfig struct {
	Audience string `cli:"audience"`
	Job      string `cli:"job"      validate:"required"`

	// Global flags
	Debug       bool     `cli:"debug"`
	LogLevel    string   `cli:"log-level"`
	NoColor     bool     `cli:"no-color"`
	Experiments []string `cli:"experiment" normalize:"list"`
	Profile     string   `cli:"profile"`

	// API config
	DebugHTTP        bool   `cli:"debug-http"`
	AgentAccessToken string `cli:"agent-access-token" validate:"required"`
	Endpoint         string `cli:"endpoint"           validate:"required"`
	NoHTTP2          bool   `cli:"no-http2"`
}

const (
	oidcTokenDescription = `Usage:

   buildkite-agent oidc token [options...]

Description:
   Requests and prints an OIDC token from Buildkite that claims the Job ID and
	 the specified audience. If no audience is specified, the endpoint's default
	 audience will be claimed.

Example:
   $ buildkite-agent oidc token --audience sts.amazonaws.com

   Prints the environment passed into the process
`
	backoffSeconds = 2
	maxAttempts    = 5
)

var OidcTokenCommand = cli.Command{
	Name:        "token",
	Usage:       "Requests and prints an OIDC token from Buildkite with the specified audience,",
	Description: oidcTokenDescription,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "audience",
			Value: "",
			Usage: "The audience that will consume the OIDC token",
		},
		cli.StringFlag{
			Name:   "job",
			Value:  "",
			Usage:  "Buildkite Job Id to claim in the OIDC token",
			EnvVar: "BUILDKITE_JOB_ID",
		},

		// API Flags
		AgentAccessTokenFlag,
		EndpointFlag,
		NoHTTP2Flag,
		DebugHTTPFlag,

		// Global flags
		NoColorFlag,
		DebugFlag,
		LogLevelFlag,
		ExperimentsFlag,
		ProfileFlag,
	},
	Action: func(c *cli.Context) error {
		// The configuration will be loaded into this struct
		cfg := OidcTokenConfig{}

		loader := cliconfig.Loader{CLI: c, Config: &cfg}
		warnings, err := loader.Load()
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err)
			os.Exit(1)
		}

		l := CreateLogger(&cfg)

		// Now that we have a logger, log out the warnings that loading config generated
		for _, warning := range warnings {
			l.Warn("%s", warning)
		}

		// Setup any global configuration options
		done := HandleGlobalFlags(l, cfg)
		defer done()

		// Create the API client
		client := api.NewClient(l, loadAPIClientConfig(cfg, "AgentAccessToken"))

		// Find the meta data value
		var token *api.OidcToken
		var resp *api.Response

		if err := roko.NewRetrier(
			roko.WithMaxAttempts(maxAttempts),
			roko.WithStrategy(roko.Exponential(backoffSeconds*time.Second, 0)),
		).Do(func(r *roko.Retrier) error {
			var audience []string
			if len(cfg.Audience) > 0 {
				audience = []string{cfg.Audience}
			}

			token, resp, err = client.OidcToken(cfg.Job, audience...)
			if resp != nil {
				fmt.Println(resp.StatusCode, r.ShouldGiveUp())

				switch resp.StatusCode {
				// Don't bother retrying if the response was one of these statuses
				case http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden, http.StatusUnprocessableEntity:
					r.Break()
					return err
				}
			}

			if err != nil {
				l.Warn("%s (%s)", err, r)
			}

			return err
		}); err != nil {
			if len(cfg.Audience) > 0 {
				l.Error("Could not obtain OIDC token for audience %s", cfg.Audience)
			} else {
				l.Error("Could not obtain OIDC token for default audience")
			}
			return err
		}

		fmt.Println(token.Token)
		return nil
	},
}
<|endoftext|>