<commit_before>// Copyright (c) 2015 Andy Leap, Google
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.

// Run the shared test suite from https://github.com/microformats/tests

package microformats_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/url"
	"path/filepath"
	"reflect"
	"testing"

	"willnorris.com/go/microformats"
)

// list of tests built using:
//     find testdata/tests/microformats-v2 -name "*.html" | grep -v change-log | sed -E 's/^testdata\/tests\/(.+)\.html$/\1/'
var tests = []string{
	"microformats-v2/h-adr/geo",
	"microformats-v2/h-adr/geourl",
	"microformats-v2/h-adr/justaname",
	"microformats-v2/h-adr/simpleproperties",
	// "microformats-v2/h-as-note/note",
	"microformats-v2/h-card/baseurl",
	"microformats-v2/h-card/childimplied",
	"microformats-v2/h-card/extendeddescription",
	"microformats-v2/h-card/hcard",
	"microformats-v2/h-card/horghcard",
	"microformats-v2/h-card/hyperlinkedphoto",
	// "microformats-v2/h-card/impliedname",
	// "microformats-v2/h-card/impliedphoto",
	// "microformats-v2/h-card/impliedurl",
	"microformats-v2/h-card/justahyperlink",
	"microformats-v2/h-card/justaname",
	// "microformats-v2/h-card/nested",
	// "microformats-v2/h-card/p-property",
	"microformats-v2/h-card/relativeurls",
	"microformats-v2/h-entry/impliedvalue-nested",
	"microformats-v2/h-entry/justahyperlink",
	"microformats-v2/h-entry/justaname",
	// "microformats-v2/h-entry/summarycontent",
	"microformats-v2/h-entry/u-property",
	// "microformats-v2/h-entry/urlincontent",
	// "microformats-v2/h-event/ampm",
	"microformats-v2/h-event/attendees",
	"microformats-v2/h-event/combining",
	// "microformats-v2/h-event/concatenate",
	// "microformats-v2/h-event/dates",
	// "microformats-v2/h-event/dt-property",
	"microformats-v2/h-event/justahyperlink",
	"microformats-v2/h-event/justaname",
	// "microformats-v2/h-event/time",
	// "microformats-v2/h-feed/implied-title",
	// "microformats-v2/h-feed/simple",
	"microformats-v2/h-geo/abbrpattern",
	"microformats-v2/h-geo/altitude",
	"microformats-v2/h-geo/hidden",
	"microformats-v2/h-geo/justaname",
	"microformats-v2/h-geo/simpleproperties",
	"microformats-v2/h-geo/valuetitleclass",
	// "microformats-v2/h-news/all",
	// "microformats-v2/h-news/minimum",
	"microformats-v2/h-org/hyperlink",
	"microformats-v2/h-org/simple",
	"microformats-v2/h-org/simpleproperties",
	"microformats-v2/h-product/aggregate",
	"microformats-v2/h-product/justahyperlink",
	"microformats-v2/h-product/justaname",
	"microformats-v2/h-product/simpleproperties",
	// "microformats-v2/h-recipe/all",
	"microformats-v2/h-recipe/minimum",
	// "microformats-v2/h-resume/affiliation",
	"microformats-v2/h-resume/contact",
	"microformats-v2/h-resume/education",
	"microformats-v2/h-resume/justaname",
	"microformats-v2/h-resume/skill",
	"microformats-v2/h-resume/work",
	"microformats-v2/h-review/hyperlink",
	"microformats-v2/h-review/implieditem",
	"microformats-v2/h-review/item",
	"microformats-v2/h-review/justaname",
	"microformats-v2/h-review/photo",
	// "microformats-v2/h-review/vcard",
	"microformats-v2/h-review-aggregate/hevent",
	"microformats-v2/h-review-aggregate/justahyperlink",
	"microformats-v2/h-review-aggregate/simpleproperties",
	// "microformats-v2/rel/duplicate-rels",
	"microformats-v2/rel/license",
	"microformats-v2/rel/nofollow",
	"microformats-v2/rel/rel-urls",
	// "microformats-v2/rel/varying-text-duplicate-rels",
	"microformats-v2/rel/xfn-all",
	"microformats-v2/rel/xfn-elsewhere",
}

func TestSuite(t *testing.T) {
	passes := 0
	count := 0
	for _, test := range tests {
		count++
		if runTest(t, filepath.Join("testdata", "tests", test)) {
			passes++
		}
	}
	fmt.Printf("PASSING %d OF %d\n", passes, count)
}

func runTest(t *testing.T, test string) bool {
	input, err := ioutil.ReadFile(test + ".html")
	if err != nil {
		t.Fatalf("error reading file %q: %v", test+".html", err)
	}

	URL, _ := url.Parse("http://example.com/")
	data := microformats.Parse(bytes.NewReader(input), URL)

	expectedJSON, err := ioutil.ReadFile(test + ".json")
	if err != nil {
		t.Fatalf("error reading file %q: %v", test+".json", err)
	}
	want := make(map[string]interface{})
	json.Unmarshal(expectedJSON, &want)

	outputJSON, _ := json.Marshal(data)
	got := make(map[string]interface{})
	json.Unmarshal(outputJSON, &got)

	if reflect.DeepEqual(got, want) {
		fmt.Printf("PASS: %s\n", test)
		return true
	}

	fmt.Printf("FAIL: %s\ngot: %v\n\nwant: %v\n\n", test, got, want)
	t.Fail()
	return false
}
<commit_msg>handle json errors in testsuite_test.go<commit_after>// Copyright (c) 2015 Andy Leap, Google
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.

// Run the shared test suite from https://github.com/microformats/tests

package microformats_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/url"
	"path/filepath"
	"reflect"
	"testing"

	"willnorris.com/go/microformats"
)

// list of tests built using:
//     find testdata/tests/microformats-v2 -name "*.html" | grep -v change-log | sed -E 's/^testdata\/tests\/(.+)\.html$/\1/'
var tests = []string{
	"microformats-v2/h-adr/geo",
	"microformats-v2/h-adr/geourl",
	"microformats-v2/h-adr/justaname",
	"microformats-v2/h-adr/simpleproperties",
	// "microformats-v2/h-as-note/note",
	"microformats-v2/h-card/baseurl",
	"microformats-v2/h-card/childimplied",
	"microformats-v2/h-card/extendeddescription",
	"microformats-v2/h-card/hcard",
	"microformats-v2/h-card/horghcard",
	"microformats-v2/h-card/hyperlinkedphoto",
	// "microformats-v2/h-card/impliedname",
	// "microformats-v2/h-card/impliedphoto",
	// "microformats-v2/h-card/impliedurl",
	"microformats-v2/h-card/justahyperlink",
	"microformats-v2/h-card/justaname",
	// "microformats-v2/h-card/nested",
	// "microformats-v2/h-card/p-property",
	"microformats-v2/h-card/relativeurls",
	"microformats-v2/h-entry/impliedvalue-nested",
	"microformats-v2/h-entry/justahyperlink",
	"microformats-v2/h-entry/justaname",
	// "microformats-v2/h-entry/summarycontent",
	"microformats-v2/h-entry/u-property",
	// "microformats-v2/h-entry/urlincontent",
	// "microformats-v2/h-event/ampm",
	"microformats-v2/h-event/attendees",
	"microformats-v2/h-event/combining",
	// "microformats-v2/h-event/concatenate",
	// "microformats-v2/h-event/dates",
	// "microformats-v2/h-event/dt-property",
	"microformats-v2/h-event/justahyperlink",
	"microformats-v2/h-event/justaname",
	// "microformats-v2/h-event/time",
	// "microformats-v2/h-feed/implied-title",
	// "microformats-v2/h-feed/simple",
	"microformats-v2/h-geo/abbrpattern",
	"microformats-v2/h-geo/altitude",
	"microformats-v2/h-geo/hidden",
	"microformats-v2/h-geo/justaname",
	"microformats-v2/h-geo/simpleproperties",
	"microformats-v2/h-geo/valuetitleclass",
	// "microformats-v2/h-news/all",
	// "microformats-v2/h-news/minimum",
	"microformats-v2/h-org/hyperlink",
	"microformats-v2/h-org/simple",
	"microformats-v2/h-org/simpleproperties",
	"microformats-v2/h-product/aggregate",
	"microformats-v2/h-product/justahyperlink",
	"microformats-v2/h-product/justaname",
	"microformats-v2/h-product/simpleproperties",
	// "microformats-v2/h-recipe/all",
	"microformats-v2/h-recipe/minimum",
	// "microformats-v2/h-resume/affiliation",
	"microformats-v2/h-resume/contact",
	"microformats-v2/h-resume/education",
	"microformats-v2/h-resume/justaname",
	"microformats-v2/h-resume/skill",
	"microformats-v2/h-resume/work",
	"microformats-v2/h-review/hyperlink",
	"microformats-v2/h-review/implieditem",
	"microformats-v2/h-review/item",
	"microformats-v2/h-review/justaname",
	"microformats-v2/h-review/photo",
	// "microformats-v2/h-review/vcard",
	"microformats-v2/h-review-aggregate/hevent",
	"microformats-v2/h-review-aggregate/justahyperlink",
	"microformats-v2/h-review-aggregate/simpleproperties",
	// "microformats-v2/rel/duplicate-rels",
	"microformats-v2/rel/license",
	"microformats-v2/rel/nofollow",
	"microformats-v2/rel/rel-urls",
	// "microformats-v2/rel/varying-text-duplicate-rels",
	"microformats-v2/rel/xfn-all",
	"microformats-v2/rel/xfn-elsewhere",
}

func TestSuite(t *testing.T) {
	passes := 0
	count := 0
	for _, test := range tests {
		count++
		if runTest(t, filepath.Join("testdata", "tests", test)) {
			passes++
		}
	}
	fmt.Printf("PASSING %d OF %d\n", passes, count)
}

func runTest(t *testing.T, test string) bool {
	input, err := ioutil.ReadFile(test + ".html")
	if err != nil {
		t.Fatalf("error reading file %q: %v", test+".html", err)
	}

	URL, _ := url.Parse("http://example.com/")
	data := microformats.Parse(bytes.NewReader(input), URL)

	expectedJSON, err := ioutil.ReadFile(test + ".json")
	if err != nil {
		t.Fatalf("error reading file %q: %v", test+".json", err)
	}
	want := make(map[string]interface{})
	err = json.Unmarshal(expectedJSON, &want)
	if err != nil {
		t.Fatalf("error unmarshaling json in file %q: %v", test+".json", err)
	}

	outputJSON, _ := json.Marshal(data)
	got := make(map[string]interface{})
	err = json.Unmarshal(outputJSON, &got)
	if err != nil {
		t.Fatalf("error unmarshaling json: %v", err)
	}

	if reflect.DeepEqual(got, want) {
		fmt.Printf("PASS: %s\n", test)
		return true
	}

	fmt.Printf("FAIL: %s\ngot: %v\n\nwant: %v\n\n", test, got, want)
	t.Fail()
	return false
}
<|endoftext|>


<commit_before>// +build order

package dev_test

import (
	"fmt"
	"sync"
	"testing"

	qc "github.com/relab/gorums/dev"
	"github.com/relab/gorums/internal/leakcheck"
	"golang.org/x/net/context"
)

func TestQuorumCallOrdering(t *testing.T) {
	defer leakcheck.Check(t)
	failMsg := make(chan string)
	servers, dialOpts, stopGrpcServe, closeListeners := setup(
		t,
		[]storageServer{
			{impl: newStorageServerRequiringOrdering(failMsg)},
			{impl: newStorageServerRequiringOrdering(failMsg)},
			{impl: newStorageServerRequiringOrdering(failMsg)},
			{impl: newStorageServerRequiringOrdering(failMsg)},
			{impl: newStorageServerRequiringOrdering(failMsg)},
		},
		false,
		false,
	)
	defer closeListeners(allServers)
	defer stopGrpcServe(allServers)

	mgrOpts := []qc.ManagerOption{
		dialOpts,
	}
	mgr, err := qc.NewManager(
		servers.addrs(),
		mgrOpts...,
	)
	if err != nil {
		t.Fatalf("%v", err)
	}
	defer mgr.Close()

	ids := mgr.NodeIDs()

	// Quorum spec: rq=1 (not used), wq=3, n=5.
	qspec := NewStorageQSpec(1, 3)

	config, err := mgr.NewConfiguration(ids, qspec)
	if err != nil {
		t.Fatalf("error creating config: %v", err)
	}

	state := &qc.State{
		Value:     "42",
		Timestamp: 1, // Servers start at 0.
	}

	const maxNumberOfQCs = 1 << 18

	// Test description:
	//
	// Sequentially call Write with an increasing timestamp.
	//
	// n=5, wq=3 -> Write QC will return after three replies.
	//
	// Do _not_ cancel context to allow the two remaining goroutines per
	// call to be handled on the servers.
	//
	// Servers will report error if a message received out-of-order.
	for i := 1; i < maxNumberOfQCs; i++ {
		select {
		case err := <-failMsg:
			t.Fatalf(err)
		default:
			_, err := config.Write(context.Background(), state)
			if err != nil {
				t.Fatalf("write quorum call error: %v", err)
			}
			state.Timestamp++
		}
	}
}

// StorageServerRequiringOrdering represents a single state storage that
// requires strict ordering of received messages.
type storageServerRequiringOrdering struct {
	mu        sync.Mutex
	state     qc.State
	writeResp qc.WriteResponse
	failMsg   chan string
}

// NewStorageBasic returns a new basic storage server.
func newStorageServerRequiringOrdering(failMsg chan string) *storageServerRequiringOrdering {
	return &storageServerRequiringOrdering{failMsg: failMsg}
}

// Read implements the Read method.
func (s *storageServerRequiringOrdering) Read(ctx context.Context, rq *qc.ReadRequest) (*qc.State, error) {
	panic("not implemented")
}

// ReadCorrectable implements the ReadCorrectable method.
func (s *storageServerRequiringOrdering) ReadCorrectable(ctx context.Context, rq *qc.ReadRequest) (*qc.State, error) {
	panic("not implemented")
}

// ReadFuture implements the ReadFuture method.
func (s *storageServerRequiringOrdering) ReadFuture(ctx context.Context, rq *qc.ReadRequest) (*qc.State, error) {
	panic("not implemented")
}

// ReadCustomReturn implements the ReadCustomReturn method.
func (s *storageServerRequiringOrdering) ReadCustomReturn(ctx context.Context, rq *qc.ReadRequest) (*qc.State, error) {
	panic("not implemented")
}

// Write implements the Write method.
func (s *storageServerRequiringOrdering) Write(ctx context.Context, state *qc.State) (*qc.WriteResponse, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if state.Timestamp != s.state.Timestamp+1 {
		err := fmt.Sprintf(
			"server: message received out-of-order: got: %d, expected: %d",
			state.Timestamp, s.state.Timestamp+1,
		)
		s.failMsg <- err
	} else {
		s.state = *state
	}
	return &s.writeResp, nil
}

// WriteFuture implements the WriteFuture method.
func (s *storageServerRequiringOrdering) WriteFuture(ctx context.Context, state *qc.State) (*qc.WriteResponse, error) {
	panic("not implemented")
}

// WritePerNode implements the WritePerNode method.
func (s *storageServerRequiringOrdering) WritePerNode(ctx context.Context, state *qc.State) (*qc.WriteResponse, error) {
	panic("not implemented")
}

// WriteAsync implements the WriteAsync method from the StorageServer interface.
func (s *storageServerRequiringOrdering) WriteAsync(stream qc.Storage_WriteAsyncServer) error {
	return nil
}

// ReadNoQC implements the ReadNoQC method from the StorageServer interface.
func (s *storageServerRequiringOrdering) ReadNoQC(ctx context.Context, rq *qc.ReadRequest) (*qc.State, error) {
	panic("not implemented")
}

// ReadCorrectableStream implements the ReadCorrectableStream method from the StorageServer interface.
func (s *storageServerRequiringOrdering) ReadCorrectableStream(rq *qc.ReadRequest, rrts qc.Storage_ReadCorrectableStreamServer) error {
	panic("not implemented")
}

// ReadExecuted returns when r has has completed a read.
func (s *storageServerRequiringOrdering) ReadExecuted() {
	panic("not implemented")
}

// WriteExecuted returns when r has has completed a write.
func (s *storageServerRequiringOrdering) WriteExecuted() {
	panic("not implemented")
}
<commit_msg>fixed failing ordering test; required WithDialTimeout as mgr opt<commit_after>// +build order

package dev_test

import (
	"fmt"
	"sync"
	"testing"
	"time"

	qc "github.com/relab/gorums/dev"
	"github.com/relab/gorums/internal/leakcheck"
	"golang.org/x/net/context"
)

func TestQuorumCallOrdering(t *testing.T) {
	defer leakcheck.Check(t)
	failMsg := make(chan string)
	servers, dialOpts, stopGrpcServe, closeListeners := setup(
		t,
		[]storageServer{
			{impl: newStorageServerRequiringOrdering(failMsg)},
			{impl: newStorageServerRequiringOrdering(failMsg)},
			{impl: newStorageServerRequiringOrdering(failMsg)},
			{impl: newStorageServerRequiringOrdering(failMsg)},
			{impl: newStorageServerRequiringOrdering(failMsg)},
		},
		false,
		false,
	)
	defer closeListeners(allServers)
	defer stopGrpcServe(allServers)

	mgr, err := qc.NewManager(servers.addrs(), dialOpts, qc.WithDialTimeout(time.Second))
	if err != nil {
		t.Fatalf("%v", err)
	}
	defer mgr.Close()

	ids := mgr.NodeIDs()

	// Quorum spec: rq=1 (not used), wq=3, n=5.
	qspec := NewStorageQSpec(1, 3)

	config, err := mgr.NewConfiguration(ids, qspec)
	if err != nil {
		t.Fatalf("error creating config: %v", err)
	}

	state := &qc.State{
		Value:     "42",
		Timestamp: 1, // Servers start at 0.
	}

	const maxNumberOfQCs = 1 << 18

	// Test description:
	//
	// Sequentially call Write with an increasing timestamp.
	//
	// n=5, wq=3 -> Write QC will return after three replies.
	//
	// Do _not_ cancel context to allow the two remaining goroutines per
	// call to be handled on the servers.
	//
	// Servers will report error if a message received out-of-order.
	for i := 1; i < maxNumberOfQCs; i++ {
		select {
		case err := <-failMsg:
			t.Fatalf(err)
		default:
			_, err := config.Write(context.Background(), state)
			if err != nil {
				t.Fatalf("write quorum call error: %v", err)
			}
			state.Timestamp++
		}
	}
}

// StorageServerRequiringOrdering represents a single state storage that
// requires strict ordering of received messages.
type storageServerRequiringOrdering struct {
	mu        sync.Mutex
	state     qc.State
	writeResp qc.WriteResponse
	failMsg   chan string
}

// NewStorageBasic returns a new basic storage server.
func newStorageServerRequiringOrdering(failMsg chan string) *storageServerRequiringOrdering {
	return &storageServerRequiringOrdering{failMsg: failMsg}
}

// Read implements the Read method.
func (s *storageServerRequiringOrdering) Read(ctx context.Context, rq *qc.ReadRequest) (*qc.State, error) {
	panic("not implemented")
}

// ReadCorrectable implements the ReadCorrectable method.
func (s *storageServerRequiringOrdering) ReadCorrectable(ctx context.Context, rq *qc.ReadRequest) (*qc.State, error) {
	panic("not implemented")
}

// ReadFuture implements the ReadFuture method.
func (s *storageServerRequiringOrdering) ReadFuture(ctx context.Context, rq *qc.ReadRequest) (*qc.State, error) {
	panic("not implemented")
}

// ReadCustomReturn implements the ReadCustomReturn method.
func (s *storageServerRequiringOrdering) ReadCustomReturn(ctx context.Context, rq *qc.ReadRequest) (*qc.State, error) {
	panic("not implemented")
}

// Write implements the Write method.
func (s *storageServerRequiringOrdering) Write(ctx context.Context, state *qc.State) (*qc.WriteResponse, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if state.Timestamp != s.state.Timestamp+1 {
		err := fmt.Sprintf(
			"server: message received out-of-order: got: %d, expected: %d",
			state.Timestamp, s.state.Timestamp+1,
		)
		s.failMsg <- err
	} else {
		s.state = *state
	}
	return &s.writeResp, nil
}

// WriteFuture implements the WriteFuture method.
func (s *storageServerRequiringOrdering) WriteFuture(ctx context.Context, state *qc.State) (*qc.WriteResponse, error) {
	panic("not implemented")
}

// WritePerNode implements the WritePerNode method.
func (s *storageServerRequiringOrdering) WritePerNode(ctx context.Context, state *qc.State) (*qc.WriteResponse, error) {
	panic("not implemented")
}

// WriteAsync implements the WriteAsync method from the StorageServer interface.
func (s *storageServerRequiringOrdering) WriteAsync(stream qc.Storage_WriteAsyncServer) error {
	return nil
}

// ReadNoQC implements the ReadNoQC method from the StorageServer interface.
func (s *storageServerRequiringOrdering) ReadNoQC(ctx context.Context, rq *qc.ReadRequest) (*qc.State, error) {
	panic("not implemented")
}

// ReadCorrectableStream implements the ReadCorrectableStream method from the StorageServer interface.
func (s *storageServerRequiringOrdering) ReadCorrectableStream(rq *qc.ReadRequest, rrts qc.Storage_ReadCorrectableStreamServer) error {
	panic("not implemented")
}

// ReadExecuted returns when r has has completed a read.
func (s *storageServerRequiringOrdering) ReadExecuted() {
	panic("not implemented")
}

// WriteExecuted returns when r has has completed a write.
func (s *storageServerRequiringOrdering) WriteExecuted() {
	panic("not implemented")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/node"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/rsync"
	"github.com/lxc/lxd/lxd/state"
	storagePools "github.com/lxc/lxd/lxd/storage"
	storageDrivers "github.com/lxc/lxd/lxd/storage/drivers"
	"github.com/lxc/lxd/shared"
)

func daemonStorageMount(s *state.State) error {
	var storageBackups string
	var storageImages string
	err := s.Node.Transaction(func(tx *db.NodeTx) error {
		nodeConfig, err := node.ConfigLoad(tx)
		if err != nil {
			return err
		}

		storageBackups = nodeConfig.StorageBackupsVolume()
		storageImages = nodeConfig.StorageImagesVolume()

		return nil
	})
	if err != nil {
		return err
	}

	mount := func(storageType string, source string) error {
		// Parse the source.
		fields := strings.Split(source, "/")
		if len(fields) != 2 {
			return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>")
		}

		poolName := fields[0]
		volumeName := fields[1]

		pool, err := storagePools.GetPoolByName(s, poolName)
		if err != nil {
			return err
		}

		// Mount volume.
		_, err = pool.MountCustomVolume(project.Default, volumeName, nil)
		if err != nil {
			return errors.Wrapf(err, "Failed to mount storage volume %q", source)
		}

		return nil
	}

	if storageBackups != "" {
		err := mount("backups", storageBackups)
		if err != nil {
			return errors.Wrap(err, "Failed to mount backups storage")
		}
	}

	if storageImages != "" {
		err := mount("images", storageImages)
		if err != nil {
			return errors.Wrap(err, "Failed to mount images storage")
		}
	}

	return nil
}

func daemonStorageValidate(s *state.State, target string) error {
	// Check syntax.
	if target == "" {
		return nil
	}

	fields := strings.Split(target, "/")
	if len(fields) != 2 {
		return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>")
	}

	poolName := fields[0]
	volumeName := fields[1]

	// Validate pool exists.
	poolID, dbPool, err := s.Cluster.GetStoragePool(poolName)
	if err != nil {
		return errors.Wrapf(err, "Unable to load storage pool %q", poolName)
	}

	// Validate pool driver (can't be CEPH or CEPHFS).
	if dbPool.Driver == "ceph" || dbPool.Driver == "cephfs" {
		return fmt.Errorf("Server storage volumes cannot be stored on Ceph")
	}

	// Confirm volume exists.
	_, _, err = s.Cluster.GetLocalStoragePoolVolume(project.Default, volumeName, db.StoragePoolVolumeTypeCustom, poolID)
	if err != nil {
		return errors.Wrapf(err, "Unable to load storage volume %q", target)
	}

	snapshots, err := s.Cluster.GetLocalStoragePoolVolumeSnapshotsWithType(project.Default, volumeName, db.StoragePoolVolumeTypeCustom, poolID)
	if err != nil {
		return errors.Wrapf(err, "Unable to load storage volume snapshots %q", target)
	}

	if len(snapshots) != 0 {
		return fmt.Errorf("Storage volumes for use by LXD itself cannot have snapshots")
	}

	pool, err := storagePools.GetPoolByName(s, poolName)
	if err != nil {
		return err
	}

	// Mount volume.
	ourMount, err := pool.MountCustomVolume(project.Default, volumeName, nil)
	if err != nil {
		return errors.Wrapf(err, "Failed to mount storage volume %q", target)
	}
	if ourMount {
		defer pool.UnmountCustomVolume(project.Default, volumeName, nil)
	}

	// Validate volume is empty (ignore lost+found).
	volStorageName := project.StorageVolume(project.Default, volumeName)
	mountpoint := storageDrivers.GetVolumeMountPath(poolName, storageDrivers.VolumeTypeCustom, volStorageName)

	entries, err := ioutil.ReadDir(mountpoint)
	if err != nil {
		return errors.Wrapf(err, "Failed to list %q", mountpoint)
	}

	for _, entry := range entries {
		entryName := entry.Name()

		// Don't fail on clean ext4 volumes.
		if entryName == "lost+found" {
			continue
		}

		// Don't fail on systems with snapdir=visible.
		if entryName == ".zfs" {
			continue
		}

		return fmt.Errorf("Storage volume %q isn't empty", target)
	}

	return nil
}

func daemonStorageMove(s *state.State, storageType string, target string) error {
	destPath := shared.VarPath(storageType)

	// Track down the current storage.
	var sourcePool string
	var sourceVolume string

	sourcePath, err := os.Readlink(destPath)
	if err != nil {
		sourcePath = destPath
	} else {
		fields := strings.Split(sourcePath, "/")
		sourcePool = fields[len(fields)-3]
		sourceVolume = fields[len(fields)-1]
	}

	moveContent := func(source string, target string) error {
		// Copy the content.
		_, err := rsync.LocalCopy(source, target, "", false)
		if err != nil {
			return err
		}

		// Remove the source content.
		entries, err := ioutil.ReadDir(source)
		if err != nil {
			return err
		}

		for _, entry := range entries {
			err := os.RemoveAll(filepath.Join(source, entry.Name()))
			if err != nil {
				return err
			}
		}

		return nil
	}

	// Deal with unsetting.
	if target == "" {
		// Things already look correct.
		if sourcePath == destPath {
			return nil
		}

		// Remove the symlink.
		err = os.Remove(destPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to delete storage symlink at %q", destPath)
		}

		// Re-create as a directory.
		err = os.MkdirAll(destPath, 0700)
		if err != nil {
			return errors.Wrapf(err, "Failed to create directory %q", destPath)
		}

		// Move the data across.
		err = moveContent(sourcePath, destPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to move data over to directory %q", destPath)
		}

		pool, err := storagePools.GetPoolByName(s, sourcePool)
		if err != nil {
			return err
		}

		// Unmount old volume.
		projectName, sourceVolumeName := project.StorageVolumeParts(sourceVolume)
		_, err = pool.UnmountCustomVolume(projectName, sourceVolumeName, nil)
		if err != nil {
			return errors.Wrapf(err, `Failed to umount storage volume "%s/%s"`, sourcePool, sourceVolumeName)
		}

		return nil
	}

	// Parse the target.
	fields := strings.Split(target, "/")
	if len(fields) != 2 {
		return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>")
	}

	poolName := fields[0]
	volumeName := fields[1]

	pool, err := storagePools.GetPoolByName(s, poolName)
	if err != nil {
		return err
	}

	// Mount volume.
	_, err = pool.MountCustomVolume(project.Default, volumeName, nil)
	if err != nil {
		return errors.Wrapf(err, "Failed to mount storage volume %q", target)
	}

	// Set ownership & mode.
	volStorageName := project.StorageVolume(project.Default, volumeName)
	mountpoint := storageDrivers.GetVolumeMountPath(poolName, storageDrivers.VolumeTypeCustom, volStorageName)
	destPath = mountpoint

	err = os.Chmod(mountpoint, 0700)
	if err != nil {
		return errors.Wrapf(err, "Failed to set permissions on %q", mountpoint)
	}

	err = os.Chown(mountpoint, 0, 0)
	if err != nil {
		return errors.Wrapf(err, "Failed to set ownership on %q", mountpoint)
	}

	// Handle changes.
	if sourcePath != shared.VarPath(storageType) {
		// Remove the symlink.
		err := os.Remove(shared.VarPath(storageType))
		if err != nil {
			return errors.Wrapf(err, "Failed to remove the new symlink at %q", shared.VarPath(storageType))
		}

		// Create the new symlink.
		err = os.Symlink(destPath, shared.VarPath(storageType))
		if err != nil {
			return errors.Wrapf(err, "Failed to create the new symlink at %q", shared.VarPath(storageType))
		}

		// Move the data across.
		err = moveContent(sourcePath, destPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to move data over to directory %q", destPath)
		}

		pool, err := storagePools.GetPoolByName(s, sourcePool)
		if err != nil {
			return err
		}

		// Unmount old volume.
		_, err = pool.UnmountCustomVolume(project.Default, sourceVolume, nil)
		if err != nil {
			return errors.Wrapf(err, `Failed to umount storage volume "%s/%s"`, sourcePool, sourceVolume)
		}

		return nil
	}

	sourcePath = shared.VarPath(storageType) + ".temp"

	// Rename the existing storage.
	err = os.Rename(shared.VarPath(storageType), sourcePath)
	if err != nil {
		return errors.Wrapf(err, "Failed to rename existing storage %q", shared.VarPath(storageType))
	}

	// Create the new symlink.
	err = os.Symlink(destPath, shared.VarPath(storageType))
	if err != nil {
		return errors.Wrapf(err, "Failed to create the new symlink at %q", shared.VarPath(storageType))
	}

	// Move the data across.
	err = moveContent(sourcePath, destPath)
	if err != nil {
		return errors.Wrapf(err, "Failed to move data over to directory %q", destPath)
	}

	// Remove the old data.
	err = os.RemoveAll(sourcePath)
	if err != nil {
		return errors.Wrapf(err, "Failed to cleanup old directory %q", sourcePath)
	}

	return nil
}
<commit_msg>lxd/storage: Allow ceph/cephfs for images/backups<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/node"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/rsync"
	"github.com/lxc/lxd/lxd/state"
	storagePools "github.com/lxc/lxd/lxd/storage"
	storageDrivers "github.com/lxc/lxd/lxd/storage/drivers"
	"github.com/lxc/lxd/shared"
)

func daemonStorageMount(s *state.State) error {
	var storageBackups string
	var storageImages string
	err := s.Node.Transaction(func(tx *db.NodeTx) error {
		nodeConfig, err := node.ConfigLoad(tx)
		if err != nil {
			return err
		}

		storageBackups = nodeConfig.StorageBackupsVolume()
		storageImages = nodeConfig.StorageImagesVolume()

		return nil
	})
	if err != nil {
		return err
	}

	mount := func(storageType string, source string) error {
		// Parse the source.
		fields := strings.Split(source, "/")
		if len(fields) != 2 {
			return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>")
		}

		poolName := fields[0]
		volumeName := fields[1]

		pool, err := storagePools.GetPoolByName(s, poolName)
		if err != nil {
			return err
		}

		// Mount volume.
		_, err = pool.MountCustomVolume(project.Default, volumeName, nil)
		if err != nil {
			return errors.Wrapf(err, "Failed to mount storage volume %q", source)
		}

		return nil
	}

	if storageBackups != "" {
		err := mount("backups", storageBackups)
		if err != nil {
			return errors.Wrap(err, "Failed to mount backups storage")
		}
	}

	if storageImages != "" {
		err := mount("images", storageImages)
		if err != nil {
			return errors.Wrap(err, "Failed to mount images storage")
		}
	}

	return nil
}

func daemonStorageValidate(s *state.State, target string) error {
	// Check syntax.
	if target == "" {
		return nil
	}

	fields := strings.Split(target, "/")
	if len(fields) != 2 {
		return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>")
	}

	poolName := fields[0]
	volumeName := fields[1]

	// Validate pool exists.
	poolID, _, err := s.Cluster.GetStoragePool(poolName)
	if err != nil {
		return errors.Wrapf(err, "Unable to load storage pool %q", poolName)
	}

	// Confirm volume exists.
	_, _, err = s.Cluster.GetLocalStoragePoolVolume(project.Default, volumeName, db.StoragePoolVolumeTypeCustom, poolID)
	if err != nil {
		return errors.Wrapf(err, "Unable to load storage volume %q", target)
	}

	snapshots, err := s.Cluster.GetLocalStoragePoolVolumeSnapshotsWithType(project.Default, volumeName, db.StoragePoolVolumeTypeCustom, poolID)
	if err != nil {
		return errors.Wrapf(err, "Unable to load storage volume snapshots %q", target)
	}

	if len(snapshots) != 0 {
		return fmt.Errorf("Storage volumes for use by LXD itself cannot have snapshots")
	}

	pool, err := storagePools.GetPoolByName(s, poolName)
	if err != nil {
		return err
	}

	// Mount volume.
	ourMount, err := pool.MountCustomVolume(project.Default, volumeName, nil)
	if err != nil {
		return errors.Wrapf(err, "Failed to mount storage volume %q", target)
	}
	if ourMount {
		defer pool.UnmountCustomVolume(project.Default, volumeName, nil)
	}

	// Validate volume is empty (ignore lost+found).
	volStorageName := project.StorageVolume(project.Default, volumeName)
	mountpoint := storageDrivers.GetVolumeMountPath(poolName, storageDrivers.VolumeTypeCustom, volStorageName)

	entries, err := ioutil.ReadDir(mountpoint)
	if err != nil {
		return errors.Wrapf(err, "Failed to list %q", mountpoint)
	}

	for _, entry := range entries {
		entryName := entry.Name()

		// Don't fail on clean ext4 volumes.
		if entryName == "lost+found" {
			continue
		}

		// Don't fail on systems with snapdir=visible.
		if entryName == ".zfs" {
			continue
		}

		return fmt.Errorf("Storage volume %q isn't empty", target)
	}

	return nil
}

func daemonStorageMove(s *state.State, storageType string, target string) error {
	destPath := shared.VarPath(storageType)

	// Track down the current storage.
	var sourcePool string
	var sourceVolume string

	sourcePath, err := os.Readlink(destPath)
	if err != nil {
		sourcePath = destPath
	} else {
		fields := strings.Split(sourcePath, "/")
		sourcePool = fields[len(fields)-3]
		sourceVolume = fields[len(fields)-1]
	}

	moveContent := func(source string, target string) error {
		// Copy the content.
		_, err := rsync.LocalCopy(source, target, "", false)
		if err != nil {
			return err
		}

		// Remove the source content.
		entries, err := ioutil.ReadDir(source)
		if err != nil {
			return err
		}

		for _, entry := range entries {
			err := os.RemoveAll(filepath.Join(source, entry.Name()))
			if err != nil {
				return err
			}
		}

		return nil
	}

	// Deal with unsetting.
	if target == "" {
		// Things already look correct.
		if sourcePath == destPath {
			return nil
		}

		// Remove the symlink.
		err = os.Remove(destPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to delete storage symlink at %q", destPath)
		}

		// Re-create as a directory.
		err = os.MkdirAll(destPath, 0700)
		if err != nil {
			return errors.Wrapf(err, "Failed to create directory %q", destPath)
		}

		// Move the data across.
		err = moveContent(sourcePath, destPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to move data over to directory %q", destPath)
		}

		pool, err := storagePools.GetPoolByName(s, sourcePool)
		if err != nil {
			return err
		}

		// Unmount old volume.
		projectName, sourceVolumeName := project.StorageVolumeParts(sourceVolume)
		_, err = pool.UnmountCustomVolume(projectName, sourceVolumeName, nil)
		if err != nil {
			return errors.Wrapf(err, `Failed to umount storage volume "%s/%s"`, sourcePool, sourceVolumeName)
		}

		return nil
	}

	// Parse the target.
	fields := strings.Split(target, "/")
	if len(fields) != 2 {
		return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>")
	}

	poolName := fields[0]
	volumeName := fields[1]

	pool, err := storagePools.GetPoolByName(s, poolName)
	if err != nil {
		return err
	}

	// Mount volume.
	_, err = pool.MountCustomVolume(project.Default, volumeName, nil)
	if err != nil {
		return errors.Wrapf(err, "Failed to mount storage volume %q", target)
	}

	// Set ownership & mode.
	volStorageName := project.StorageVolume(project.Default, volumeName)
	mountpoint := storageDrivers.GetVolumeMountPath(poolName, storageDrivers.VolumeTypeCustom, volStorageName)
	destPath = mountpoint

	err = os.Chmod(mountpoint, 0700)
	if err != nil {
		return errors.Wrapf(err, "Failed to set permissions on %q", mountpoint)
	}

	err = os.Chown(mountpoint, 0, 0)
	if err != nil {
		return errors.Wrapf(err, "Failed to set ownership on %q", mountpoint)
	}

	// Handle changes.
	if sourcePath != shared.VarPath(storageType) {
		// Remove the symlink.
		err := os.Remove(shared.VarPath(storageType))
		if err != nil {
			return errors.Wrapf(err, "Failed to remove the new symlink at %q", shared.VarPath(storageType))
		}

		// Create the new symlink.
		err = os.Symlink(destPath, shared.VarPath(storageType))
		if err != nil {
			return errors.Wrapf(err, "Failed to create the new symlink at %q", shared.VarPath(storageType))
		}

		// Move the data across.
		err = moveContent(sourcePath, destPath)
		if err != nil {
			return errors.Wrapf(err, "Failed to move data over to directory %q", destPath)
		}

		pool, err := storagePools.GetPoolByName(s, sourcePool)
		if err != nil {
			return err
		}

		// Unmount old volume.
		_, err = pool.UnmountCustomVolume(project.Default, sourceVolume, nil)
		if err != nil {
			return errors.Wrapf(err, `Failed to umount storage volume "%s/%s"`, sourcePool, sourceVolume)
		}

		return nil
	}

	sourcePath = shared.VarPath(storageType) + ".temp"

	// Rename the existing storage.
	err = os.Rename(shared.VarPath(storageType), sourcePath)
	if err != nil {
		return errors.Wrapf(err, "Failed to rename existing storage %q", shared.VarPath(storageType))
	}

	// Create the new symlink.
	err = os.Symlink(destPath, shared.VarPath(storageType))
	if err != nil {
		return errors.Wrapf(err, "Failed to create the new symlink at %q", shared.VarPath(storageType))
	}

	// Move the data across.
	err = moveContent(sourcePath, destPath)
	if err != nil {
		return errors.Wrapf(err, "Failed to move data over to directory %q", destPath)
	}

	// Remove the old data.
	err = os.RemoveAll(sourcePath)
	if err != nil {
		return errors.Wrapf(err, "Failed to cleanup old directory %q", sourcePath)
	}

	return nil
}
<|endoftext|>


<commit_before>package m_etcd

import (
	"encoding/json"
	"os"
	"path"
	"strings"
	"sync"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/coreos/go-etcd/etcd"
	"github.com/lytics/metafora"
)

const (
	// etcd/Response.Action values
	actionCreated = "create"
	actionExpire  = "expire"
	actionDelete  = "delete"
	actionCAD     = "compareAndDelete"
)

var (
	DefaultNodePathTTL uint64 = 20 // seconds

	// etcd actions signifying a claim key was released
	releaseActions = map[string]bool{
		actionExpire: true,
		actionDelete: true,
		actionCAD:    true,
	}
)

type ownerValue struct {
	Node string `json:"node"`
}

type watcher struct {
	cordCtx      metafora.CoordinatorContext
	path         string
	responseChan chan *etcd.Response // coordinator watches for changes
	errorChan    chan error          // coordinator watches for errors
	stopChan     chan bool           // closed to signal etcd's Watch to stop
	client       *etcd.Client
	running      bool       // only set in watch(); only read by watching()
	m            sync.Mutex // running requires synchronization
}

// watch emits etcd responses over responseChan until stopChan is closed.
func (w *watcher) watch() {
	var err error

	// Teardown watch state when watch() returns and send err
	defer func() {
		w.m.Lock()
		w.running = false
		w.m.Unlock()
		w.errorChan <- err
	}()

	const sorted = true
	const recursive = true
	var resp *etcd.Response
	var rawResp *etcd.RawResponse

startWatch:
	for {
		// Get existing tasks
		resp, err = w.client.Get(w.path, sorted, recursive)
		if err != nil {
			w.cordCtx.Log(metafora.LogLevelError, "%s Error getting the existing tasks: %v", w.path, err)
			return
		}

		// Start watching at the index the Get retrieved since we've retrieved all
		// tasks up to that point.
		index := resp.EtcdIndex

		// Act like existing keys are newly created
		for _, node := range resp.Node.Nodes {
			if node.ModifiedIndex > index {
				// Record the max modified index to keep Watch from picking up redundant events
				index = node.ModifiedIndex
			}
			select {
			case <-w.stopChan:
				return
			case w.responseChan <- &etcd.Response{Action: "create", Node: node}:
			}
		}

		// Start blocking watch
		for {
			// Start the blocking watch after the last response's index.
			rawResp, err = w.client.RawWatch(w.path, index+1, recursive, nil, w.stopChan)
			if err != nil {
				if err == etcd.ErrWatchStoppedByUser {
					// This isn't actually an error, the stopChan was closed. Time to stop!
					err = nil
				} else {
					// RawWatch errors should be treated as fatal since it internally
					// retries on network problems, and recoverable Etcd errors aren't
					// parsed until rawResp.Unmarshal is called later.
					w.cordCtx.Log(metafora.LogLevelError, "%s Unrecoverable watch error: %v", w.path, err)
				}
				return
			}

			if len(rawResp.Body) == 0 {
				// This is a bug in Go's HTTP + go-etcd + etcd which causes the
				// connection to timeout perdiocally and need to be restarted *after*
				// closing idle connections.
				w.cordCtx.Log(metafora.LogLevelDebug, "%s Watch response empty; restarting watch", w.path)
				transport.CloseIdleConnections()
				continue
			}

			if resp, err = rawResp.Unmarshal(); err != nil {
				if ee, ok := err.(*etcd.EtcdError); ok {
					if ee.ErrorCode == EcodeExpiredIndex {
						w.cordCtx.Log(metafora.LogLevelDebug, "%s Too many events have happened since index was updated. Restarting watch.", w.path)
						// We need to retrieve all existing tasks to update our index
						// without potentially missing some events.
						continue startWatch
					}
				}
				w.cordCtx.Log(metafora.LogLevelError, "%s Unexpected error unmarshalling etcd response: %+v", w.path, err)
				return
			}

			select {
			case w.responseChan <- resp:
			case <-w.stopChan:
				return
			}

			index = resp.Node.ModifiedIndex
		}
	}
}

// ensureWatching starts watching etcd in a goroutine if it's not already running.
func (w *watcher) ensureWatching() {
	w.m.Lock()
	defer w.m.Unlock()
	if !w.running {
		// Initialize watch state here; will be updated by watch()
		w.running = true
		w.stopChan = make(chan bool)
		go w.watch()
	}
}

// stops the watching goroutine.
func (w *watcher) stop() {
	w.m.Lock()
	defer w.m.Unlock()
	if w.running {
		select {
		case <-w.stopChan:
			// already stopped, let's avoid panic()s
		default:
			close(w.stopChan)
		}
	}
}

type EtcdCoordinator struct {
	Client    *etcd.Client
	cordCtx   metafora.CoordinatorContext
	namespace string
	taskPath  string

	taskWatcher *watcher
	ClaimTTL    uint64 // seconds

	NodeID         string
	nodePath       string
	nodePathTTL    uint64
	commandPath    string
	commandWatcher *watcher

	taskManager *taskManager

	// Close() sends nodeRefresher a chan to close when it has exited.
	stopNode chan struct{}
	closeL   sync.Mutex // only process one Close() call at once
	closed   bool
}

// NewEtcdCoordinator creates a new Metafora Coordinator implementation using
// etcd as the broker. If no node ID is specified, a unique one will be
// generated.
//
// Coordinator methods will be called by the core Metafora Consumer. Calling
// Init, Close, etc. from your own code will lead to undefined behavior.
func NewEtcdCoordinator(nodeID, namespace string, client *etcd.Client) metafora.Coordinator {
	// Namespace should be an absolute path with no trailing slash
	namespace = "/" + strings.Trim(namespace, "/ ")

	if nodeID == "" {
		hn, _ := os.Hostname()
		nodeID = hn + "-" + uuid.NewRandom().String()
	}

	nodeID = strings.Trim(nodeID, "/ ")

	client.SetTransport(transport)
	client.SetConsistency(etcd.STRONG_CONSISTENCY)
	return &EtcdCoordinator{
		Client:    client,
		namespace: namespace,

		taskPath: path.Join(namespace, TasksPath),
		ClaimTTL: ClaimTTL, //default to the package constant, but allow it to be overwritten

		NodeID:      nodeID,
		nodePath:    path.Join(namespace, NodesPath, nodeID),
		nodePathTTL: DefaultNodePathTTL,
		commandPath: path.Join(namespace, NodesPath, nodeID, CommandsPath),

		stopNode: make(chan struct{}),
	}
}

// Init is called once by the consumer to provide a Logger to Coordinator
// implementations.
func (ec *EtcdCoordinator) Init(cordCtx metafora.CoordinatorContext) error {
	cordCtx.Log(metafora.LogLevelDebug, "Initializing coordinator with namespace: %s and etcd cluster: %s",
		ec.namespace, strings.Join(ec.Client.GetCluster(), ", "))

	ec.cordCtx = cordCtx

	ec.upsertDir(ec.namespace, ForeverTTL)
	ec.upsertDir(ec.taskPath, ForeverTTL)
	if _, err := ec.Client.CreateDir(ec.nodePath, ec.nodePathTTL); err != nil {
		return err
	}
	go ec.nodeRefresher()
	ec.upsertDir(ec.commandPath, ForeverTTL)

	ec.taskWatcher = &watcher{
		cordCtx:      cordCtx,
		path:         ec.taskPath,
		responseChan: make(chan *etcd.Response),
		errorChan:    make(chan error),
		client:       ec.Client,
	}

	ec.commandWatcher = &watcher{
		cordCtx:      cordCtx,
		path:         ec.commandPath,
		responseChan: make(chan *etcd.Response),
		errorChan:    make(chan error),
		client:       ec.Client,
	}

	ec.taskManager = newManager(cordCtx, ec.Client, ec.taskPath, ec.NodeID, ec.ClaimTTL)
	return nil
}

func (ec *EtcdCoordinator) upsertDir(path string, ttl uint64) {

	//hidden etcd key that isn't visible to ls commands on the directory,
	//  you have to know about it to find it :).  I'm using it to add some
	//  info about when the cluster's schema was setup.
	pathMarker := path + "/" + MetadataKey
	const sorted = false
	const recursive = false

	_, err := ec.Client.Get(path, sorted, recursive)
	if err == nil {
		return
	}

	etcdErr, ok := err.(*etcd.EtcdError)
	if ok && etcdErr.ErrorCode == EcodeKeyNotFound {
		_, err := ec.Client.CreateDir(path, ttl)
		if err != nil {
			ec.cordCtx.Log(metafora.LogLevelDebug, "Error trying to create directory. path:[%s] error:[ %v ]", path, err)
		}
		host, _ := os.Hostname()

		metadata := struct {
			Host        string `json:"host"`
			CreatedTime string `json:"created"`
			ownerValue
		}{
			Host:        host,
			CreatedTime: time.Now().String(),
			ownerValue:  ownerValue{Node: ec.NodeID},
		}
		metadataB, _ := json.Marshal(metadata)
		metadataStr := string(metadataB)
		ec.Client.Create(pathMarker, metadataStr, ttl)
	}
}

func (ec *EtcdCoordinator) nodeRefresher() {
	ttl := ec.nodePathTTL - 5 // try to have some leeway before ttl expires
	if ttl < 1 {
		ttl = 1
	}
	for {
		select {
		case <-ec.stopNode:
			return
		case <-time.After(time.Duration(ttl) * time.Second):
			if _, err := ec.Client.UpdateDir(ec.nodePath, ec.nodePathTTL); err != nil {
				ec.cordCtx.Log(metafora.LogLevelError, "Unexpected error updating node key, shutting down. Error: %v", err)

				// We're in a bad state; shut everything down
				ec.Close()
				return
			}
		}
	}
}

// Watch will do a blocking etcd watch() on taskPath until a taskId is returned.
// The return taskId isn't guaranteed to be claimable.
func (ec *EtcdCoordinator) Watch() (taskID string, err error) {
	ec.taskWatcher.ensureWatching()

watchLoop:
	for {
		select {
		case resp, ok := <-ec.taskWatcher.responseChan:
			if !ok {
				return "", nil
			}

			// Sanity check / test path invariant
			if !strings.HasPrefix(resp.Node.Key, ec.taskPath) {
				ec.cordCtx.Log(metafora.LogLevelError, "Received task from outside task path: %s", resp.Node.Key)
				continue
			}

			key := strings.Trim(resp.Node.Key, "/") // strip leading and trailing /s
			parts := strings.Split(key, "/")

			// Pickup new tasks
			if resp.Action == actionCreated && len(parts) == 3 && resp.Node.Dir {
				// Make sure it's not already claimed before returning it
				for _, n := range resp.Node.Nodes {
					if strings.HasSuffix(n.Key, OwnerMarker) {
						ec.cordCtx.Log(metafora.LogLevelDebug, "Ignoring task as it's already claimed: %s", parts[2])
						continue watchLoop
					}
				}
				ec.cordCtx.Log(metafora.LogLevelDebug, "Received new task: %s", parts[2])
				return parts[2], nil
			}

			// If a claim key is removed, try to claim the task
			if releaseActions[resp.Action] && len(parts) == 4 && parts[3] == OwnerMarker {
				ec.cordCtx.Log(metafora.LogLevelDebug, "Received released task: %s", parts[2])
				return parts[2], nil
			}

			// Ignore everything else
			ec.cordCtx.Log(metafora.LogLevelDebug, "Ignoring key in tasks: %s", resp.Node.Key)
		case err := <-ec.taskWatcher.errorChan:
			return "", err
		}
	}
}

// Claim is called by the Consumer when a Balancer has determined that a task
// ID can be claimed. Claim returns false if another consumer has already
// claimed the ID.
func (ec *EtcdCoordinator) Claim(taskID string) bool {
	return ec.taskManager.add(taskID)
}

// Release deletes the claim file.
func (ec *EtcdCoordinator) Release(taskID string) {
	const done = false
	ec.taskManager.remove(taskID, done)
}

// Done deletes the task.
func (ec *EtcdCoordinator) Done(taskID string) {
	const done = true
	ec.taskManager.remove(taskID, done)
}

// Command blocks until a command for this node is received from the broker
// by the coordinator.
func (ec *EtcdCoordinator) Command() (metafora.Command, error) {
	ec.commandWatcher.ensureWatching()

	for {
		select {
		case resp, ok := <-ec.commandWatcher.responseChan:
			if !ok {
				return nil, nil
			}

			if strings.HasSuffix(resp.Node.Key, MetadataKey) {
				// Skip metadata marker
				continue
			}

			const recurse = false
			if _, err := ec.Client.Delete(resp.Node.Key, recurse); err != nil {
				ec.cordCtx.Log(metafora.LogLevelError, "Error deleting handled command %s: %v", resp.Node.Key, err)
			}

			return metafora.UnmarshalCommand([]byte(resp.Node.Value))
		case err := <-ec.commandWatcher.errorChan:
			return nil, err
		}
	}
}

// Close stops the coordinator and causes blocking Watch and Command methods to
// return zero values.
func (ec *EtcdCoordinator) Close() {
	// Gracefully handle multiple close calls mostly to ease testing
	ec.closeL.Lock()
	defer ec.closeL.Unlock()
	if ec.closed {
		return
	}
	ec.closed = true

	ec.taskWatcher.stop()
	ec.commandWatcher.stop()
	ec.taskManager.stop()

	// Finally remove the node entry
	close(ec.stopNode)

	const recursive = true
	_, err := ec.Client.Delete(ec.nodePath, recursive)
	if err != nil {
		if eerr, ok := err.(*etcd.EtcdError); ok {
			if eerr.ErrorCode == EcodeKeyNotFound {
				// The node's TTL was up before we were able to delete it or there was
				// another problem that's already being handled.
				// The first is unlikely, the latter is already being handled, so
				// there's nothing to do here.
				return
			}
		}
		// All other errors are unexpected
		ec.cordCtx.Log(metafora.LogLevelError, "Error deleting node path %s: %v", ec.nodePath, err)
	}
}
<commit_msg>Ignoring tasks logging turned out to be extremely noisy<commit_after>package m_etcd

import (
	"encoding/json"
	"os"
	"path"
	"strings"
	"sync"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/coreos/go-etcd/etcd"
	"github.com/lytics/metafora"
)

const (
	// etcd/Response.Action values
	actionCreated = "create"
	actionExpire  = "expire"
	actionDelete  = "delete"
	actionCAD     = "compareAndDelete"
)

var (
	DefaultNodePathTTL uint64 = 20 // seconds

	// etcd actions signifying a claim key was released
	releaseActions = map[string]bool{
		actionExpire: true,
		actionDelete: true,
		actionCAD:    true,
	}
)

type ownerValue struct {
	Node string `json:"node"`
}

type watcher struct {
	cordCtx      metafora.CoordinatorContext
	path         string
	responseChan chan *etcd.Response // coordinator watches for changes
	errorChan    chan error          // coordinator watches for errors
	stopChan     chan bool           // closed to signal etcd's Watch to stop
	client       *etcd.Client
	running      bool       // only set in watch(); only read by watching()
	m            sync.Mutex // running requires synchronization
}

// watch emits etcd responses over responseChan until stopChan is closed.
func (w *watcher) watch() {
	var err error

	// Teardown watch state when watch() returns and send err
	defer func() {
		w.m.Lock()
		w.running = false
		w.m.Unlock()
		w.errorChan <- err
	}()

	const sorted = true
	const recursive = true
	var resp *etcd.Response
	var rawResp *etcd.RawResponse

startWatch:
	for {
		// Get existing tasks
		resp, err = w.client.Get(w.path, sorted, recursive)
		if err != nil {
			w.cordCtx.Log(metafora.LogLevelError, "%s Error getting the existing tasks: %v", w.path, err)
			return
		}

		// Start watching at the index the Get retrieved since we've retrieved all
		// tasks up to that point.
		index := resp.EtcdIndex

		// Act like existing keys are newly created
		for _, node := range resp.Node.Nodes {
			if node.ModifiedIndex > index {
				// Record the max modified index to keep Watch from picking up redundant events
				index = node.ModifiedIndex
			}
			select {
			case <-w.stopChan:
				return
			case w.responseChan <- &etcd.Response{Action: "create", Node: node}:
			}
		}

		// Start blocking watch
		for {
			// Start the blocking watch after the last response's index.
			rawResp, err = w.client.RawWatch(w.path, index+1, recursive, nil, w.stopChan)
			if err != nil {
				if err == etcd.ErrWatchStoppedByUser {
					// This isn't actually an error, the stopChan was closed. Time to stop!
					err = nil
				} else {
					// RawWatch errors should be treated as fatal since it internally
					// retries on network problems, and recoverable Etcd errors aren't
					// parsed until rawResp.Unmarshal is called later.
					w.cordCtx.Log(metafora.LogLevelError, "%s Unrecoverable watch error: %v", w.path, err)
				}
				return
			}

			if len(rawResp.Body) == 0 {
				// This is a bug in Go's HTTP + go-etcd + etcd which causes the
				// connection to timeout perdiocally and need to be restarted *after*
				// closing idle connections.
				w.cordCtx.Log(metafora.LogLevelDebug, "%s Watch response empty; restarting watch", w.path)
				transport.CloseIdleConnections()
				continue
			}

			if resp, err = rawResp.Unmarshal(); err != nil {
				if ee, ok := err.(*etcd.EtcdError); ok {
					if ee.ErrorCode == EcodeExpiredIndex {
						w.cordCtx.Log(metafora.LogLevelDebug, "%s Too many events have happened since index was updated. Restarting watch.", w.path)
						// We need to retrieve all existing tasks to update our index
						// without potentially missing some events.
						continue startWatch
					}
				}
				w.cordCtx.Log(metafora.LogLevelError, "%s Unexpected error unmarshalling etcd response: %+v", w.path, err)
				return
			}

			select {
			case w.responseChan <- resp:
			case <-w.stopChan:
				return
			}

			index = resp.Node.ModifiedIndex
		}
	}
}

// ensureWatching starts watching etcd in a goroutine if it's not already running.
func (w *watcher) ensureWatching() {
	w.m.Lock()
	defer w.m.Unlock()
	if !w.running {
		// Initialize watch state here; will be updated by watch()
		w.running = true
		w.stopChan = make(chan bool)
		go w.watch()
	}
}

// stops the watching goroutine.
func (w *watcher) stop() {
	w.m.Lock()
	defer w.m.Unlock()
	if w.running {
		select {
		case <-w.stopChan:
			// already stopped, let's avoid panic()s
		default:
			close(w.stopChan)
		}
	}
}

type EtcdCoordinator struct {
	Client    *etcd.Client
	cordCtx   metafora.CoordinatorContext
	namespace string
	taskPath  string

	taskWatcher *watcher
	ClaimTTL    uint64 // seconds

	NodeID         string
	nodePath       string
	nodePathTTL    uint64
	commandPath    string
	commandWatcher *watcher

	taskManager *taskManager

	// Close() sends nodeRefresher a chan to close when it has exited.
	stopNode chan struct{}
	closeL   sync.Mutex // only process one Close() call at once
	closed   bool
}

// NewEtcdCoordinator creates a new Metafora Coordinator implementation using
// etcd as the broker. If no node ID is specified, a unique one will be
// generated.
//
// Coordinator methods will be called by the core Metafora Consumer. Calling
// Init, Close, etc. from your own code will lead to undefined behavior.
func NewEtcdCoordinator(nodeID, namespace string, client *etcd.Client) metafora.Coordinator {
	// Namespace should be an absolute path with no trailing slash
	namespace = "/" + strings.Trim(namespace, "/ ")

	if nodeID == "" {
		hn, _ := os.Hostname()
		nodeID = hn + "-" + uuid.NewRandom().String()
	}

	nodeID = strings.Trim(nodeID, "/ ")

	client.SetTransport(transport)
	client.SetConsistency(etcd.STRONG_CONSISTENCY)
	return &EtcdCoordinator{
		Client:    client,
		namespace: namespace,

		taskPath: path.Join(namespace, TasksPath),
		ClaimTTL: ClaimTTL, //default to the package constant, but allow it to be overwritten

		NodeID:      nodeID,
		nodePath:    path.Join(namespace, NodesPath, nodeID),
		nodePathTTL: DefaultNodePathTTL,
		commandPath: path.Join(namespace, NodesPath, nodeID, CommandsPath),

		stopNode: make(chan struct{}),
	}
}

// Init is called once by the consumer to provide a Logger to Coordinator
// implementations.
func (ec *EtcdCoordinator) Init(cordCtx metafora.CoordinatorContext) error {
	cordCtx.Log(metafora.LogLevelDebug, "Initializing coordinator with namespace: %s and etcd cluster: %s",
		ec.namespace, strings.Join(ec.Client.GetCluster(), ", "))

	ec.cordCtx = cordCtx

	ec.upsertDir(ec.namespace, ForeverTTL)
	ec.upsertDir(ec.taskPath, ForeverTTL)
	if _, err := ec.Client.CreateDir(ec.nodePath, ec.nodePathTTL); err != nil {
		return err
	}
	go ec.nodeRefresher()
	ec.upsertDir(ec.commandPath, ForeverTTL)

	ec.taskWatcher = &watcher{
		cordCtx:      cordCtx,
		path:         ec.taskPath,
		responseChan: make(chan *etcd.Response),
		errorChan:    make(chan error),
		client:       ec.Client,
	}

	ec.commandWatcher = &watcher{
		cordCtx:      cordCtx,
		path:         ec.commandPath,
		responseChan: make(chan *etcd.Response),
		errorChan:    make(chan error),
		client:       ec.Client,
	}

	ec.taskManager = newManager(cordCtx, ec.Client, ec.taskPath, ec.NodeID, ec.ClaimTTL)
	return nil
}

func (ec *EtcdCoordinator) upsertDir(path string, ttl uint64) {

	//hidden etcd key that isn't visible to ls commands on the directory,
	//  you have to know about it to find it :).  I'm using it to add some
	//  info about when the cluster's schema was setup.
	pathMarker := path + "/" + MetadataKey
	const sorted = false
	const recursive = false

	_, err := ec.Client.Get(path, sorted, recursive)
	if err == nil {
		return
	}

	etcdErr, ok := err.(*etcd.EtcdError)
	if ok && etcdErr.ErrorCode == EcodeKeyNotFound {
		_, err := ec.Client.CreateDir(path, ttl)
		if err != nil {
			ec.cordCtx.Log(metafora.LogLevelDebug, "Error trying to create directory. path:[%s] error:[ %v ]", path, err)
		}
		host, _ := os.Hostname()

		metadata := struct {
			Host        string `json:"host"`
			CreatedTime string `json:"created"`
			ownerValue
		}{
			Host:        host,
			CreatedTime: time.Now().String(),
			ownerValue:  ownerValue{Node: ec.NodeID},
		}
		metadataB, _ := json.Marshal(metadata)
		metadataStr := string(metadataB)
		ec.Client.Create(pathMarker, metadataStr, ttl)
	}
}

func (ec *EtcdCoordinator) nodeRefresher() {
	ttl := ec.nodePathTTL - 5 // try to have some leeway before ttl expires
	if ttl < 1 {
		ttl = 1
	}
	for {
		select {
		case <-ec.stopNode:
			return
		case <-time.After(time.Duration(ttl) * time.Second):
			if _, err := ec.Client.UpdateDir(ec.nodePath, ec.nodePathTTL); err != nil {
				ec.cordCtx.Log(metafora.LogLevelError, "Unexpected error updating node key, shutting down. Error: %v", err)

				// We're in a bad state; shut everything down
				ec.Close()
				return
			}
		}
	}
}

// Watch will do a blocking etcd watch() on taskPath until a taskId is returned.
// The return taskId isn't guaranteed to be claimable.
func (ec *EtcdCoordinator) Watch() (taskID string, err error) {
	ec.taskWatcher.ensureWatching()

watchLoop:
	for {
		select {
		case resp, ok := <-ec.taskWatcher.responseChan:
			if !ok {
				return "", nil
			}

			// Sanity check / test path invariant
			if !strings.HasPrefix(resp.Node.Key, ec.taskPath) {
				ec.cordCtx.Log(metafora.LogLevelError, "Received task from outside task path: %s", resp.Node.Key)
				continue
			}

			key := strings.Trim(resp.Node.Key, "/") // strip leading and trailing /s
			parts := strings.Split(key, "/")

			// Pickup new tasks
			if resp.Action == actionCreated && len(parts) == 3 && resp.Node.Dir {
				// Make sure it's not already claimed before returning it
				for _, n := range resp.Node.Nodes {
					if strings.HasSuffix(n.Key, OwnerMarker) {
						ec.cordCtx.Log(metafora.LogLevelDebug, "Ignoring task as it's already claimed: %s", parts[2])
						continue watchLoop
					}
				}
				ec.cordCtx.Log(metafora.LogLevelDebug, "Received new task: %s", parts[2])
				return parts[2], nil
			}

			// If a claim key is removed, try to claim the task
			if releaseActions[resp.Action] && len(parts) == 4 && parts[3] == OwnerMarker {
				ec.cordCtx.Log(metafora.LogLevelDebug, "Received released task: %s", parts[2])
				return parts[2], nil
			}

			// Ignore any other key events (_metafora keys, task deletion, etc.)
		case err := <-ec.taskWatcher.errorChan:
			return "", err
		}
	}
}

// Claim is called by the Consumer when a Balancer has determined that a task
// ID can be claimed. Claim returns false if another consumer has already
// claimed the ID.
func (ec *EtcdCoordinator) Claim(taskID string) bool {
	return ec.taskManager.add(taskID)
}

// Release deletes the claim file.
func (ec *EtcdCoordinator) Release(taskID string) {
	const done = false
	ec.taskManager.remove(taskID, done)
}

// Done deletes the task.
func (ec *EtcdCoordinator) Done(taskID string) {
	const done = true
	ec.taskManager.remove(taskID, done)
}

// Command blocks until a command for this node is received from the broker
// by the coordinator.
func (ec *EtcdCoordinator) Command() (metafora.Command, error) {
	ec.commandWatcher.ensureWatching()

	for {
		select {
		case resp, ok := <-ec.commandWatcher.responseChan:
			if !ok {
				return nil, nil
			}

			if strings.HasSuffix(resp.Node.Key, MetadataKey) {
				// Skip metadata marker
				continue
			}

			const recurse = false
			if _, err := ec.Client.Delete(resp.Node.Key, recurse); err != nil {
				ec.cordCtx.Log(metafora.LogLevelError, "Error deleting handled command %s: %v", resp.Node.Key, err)
			}

			return metafora.UnmarshalCommand([]byte(resp.Node.Value))
		case err := <-ec.commandWatcher.errorChan:
			return nil, err
		}
	}
}

// Close stops the coordinator and causes blocking Watch and Command methods to
// return zero values.
func (ec *EtcdCoordinator) Close() {
	// Gracefully handle multiple close calls mostly to ease testing
	ec.closeL.Lock()
	defer ec.closeL.Unlock()
	if ec.closed {
		return
	}
	ec.closed = true

	ec.taskWatcher.stop()
	ec.commandWatcher.stop()
	ec.taskManager.stop()

	// Finally remove the node entry
	close(ec.stopNode)

	const recursive = true
	_, err := ec.Client.Delete(ec.nodePath, recursive)
	if err != nil {
		if eerr, ok := err.(*etcd.EtcdError); ok {
			if eerr.ErrorCode == EcodeKeyNotFound {
				// The node's TTL was up before we were able to delete it or there was
				// another problem that's already being handled.
				// The first is unlikely, the latter is already being handled, so
				// there's nothing to do here.
				return
			}
		}
		// All other errors are unexpected
		ec.cordCtx.Log(metafora.LogLevelError, "Error deleting node path %s: %v", ec.nodePath, err)
	}
}
<|endoftext|>


<commit_before>/*

For keeping a minimum running, perhaps when doing a routing table update, if destination hosts are all
 expired or about to expire we start more.

*/

package main

import (
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/iron-io/iron_go/cache"
	"github.com/iron-io/iron_go/worker"
	"github.com/iron-io/common"
	"log"
	"math/rand"
	// "net"
	"net/http"
	"net/url"
	"reflect"
	"strings"
	"time"
	"runtime"
	"flag"
//	"io/ioutil"
)

var config struct {
	Iron struct {
	Token      string `json:"token"`
	ProjectId  string `json:"project_id"`
} `json:"iron"`
	Logging       struct {
	To     string `json:"to"`
	Level  string `json:"level"`
	Prefix string `json:"prefix"`
}
}

//var routingTable = map[string]*Route{}
var icache = cache.New("routing-table")

func init() {

}

type Route struct {
	// TODO: Change destinations to a simple cache so it can expire entries after 55 minutes (the one we use in common?)
	Host         string `json:"host"`
	Destinations []string  `json:"destinations"`
	ProjectId    string  `json:"project_id"`
	Token        string  `json:"token"` // store this so we can queue up new workers on demand
	CodeName     string  `json:"code_name"`
}

// for adding new hosts
type Route2 struct {
	Host string `json:"host"`
	Dest string `json:"dest"`
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	log.Println("Running on", runtime.NumCPU(), "CPUs")

	var configFile string
	var env string
	flag.StringVar(&configFile, "c", "", "Config file name")
	// when this was e, it was erroring out.
	flag.StringVar(&env, "e2", "development", "environment")

	flag.Parse() // Scans the arg list and sets up flags

	// Deployer is now passing -c in since we're using upstart and it doesn't know what directory to run in
	if configFile == "" {
		configFile = "config_" + env + ".json"
	}

	common.LoadConfig("iron_mq", configFile, &config)
	common.SetLogLevel(config.Logging.Level)
	common.SetLogLocation(config.Logging.To, config.Logging.Prefix)

	icache.Settings.UseConfigMap(map[string]interface{}{"token": config.Iron.Token, "project_id": config.Iron.ProjectId})

	r := mux.NewRouter()
	s := r.Headers("Iron-Router", "").Subrouter()
	s.HandleFunc("/", AddWorker)
	r.HandleFunc("/addworker", AddWorker)

	r.HandleFunc("/", ProxyFunc)

	http.Handle("/", r)
	port := 80
	fmt.Println("listening and serving on port", port)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%v", port), nil))
}

func ProxyFunc(w http.ResponseWriter, req *http.Request) {
	fmt.Println("HOST:", req.Host)
	host := strings.Split(req.Host, ":")[0]

	// We look up the destinations in the routing table and there can be 3 possible scenarios:
	// 1) This host was never registered so we return 404
	// 2) This host has active workers so we do the proxy
	// 3) This host has no active workers so we queue one (or more) up and return a 503 or something with message that says "try again in a minute"
	//	route := routingTable[host]
	route, err := getRoute(host)
	// choose random dest
	if err != nil {
		common.SendError(w, 400, fmt.Sprintln("Host not configured or error!", err))
		return
	}
	//	if route == nil { // route.Host == "" {
	//		common.SendError(w, 400, fmt.Sprintln(w, "Host not configured!"))
	//		return
	//	}
	destIndex := rand.Intn(len(route.Destinations))
	destUrlString := route.Destinations[destIndex]
	// todo: should check if http:// already exists.
	destUrlString2 := "http://" + destUrlString
	destUrl, err := url.Parse(destUrlString2)
	if err != nil {
		fmt.Println("error!", err)
		panic(err)
	}
	fmt.Println("proxying to", destUrl)
	proxy := NewSingleHostReverseProxy(destUrl)
	err = proxy.ServeHTTP(w, req)
	if err != nil {
		fmt.Println("Error proxying!", err)
		etype := reflect.TypeOf(err)
		fmt.Println("err type:", etype)
		w.WriteHeader(http.StatusInternalServerError)
		// can't figure out how to compare types so comparing strings.... lame.
		if strings.Contains(etype.String(), "net.OpError") { // == reflect.TypeOf(net.OpError{}) { // couldn't figure out a better way to do this
			if len(route.Destinations) > 1 {
				fmt.Println("It's a network error, removing this destination from routing table.")
				route.Destinations = append(route.Destinations[:destIndex], route.Destinations[destIndex + 1:]...)
				putRoute(route)
				fmt.Println("New route:", route)
				return
			} else {
				fmt.Println("It's a network error and no other destinations available so we're going to remove it and start new task.")
				route.Destinations = append(route.Destinations[:destIndex], route.Destinations[destIndex + 1:]...)
				putRoute(route)
				fmt.Println("New route:", route)
			}
			// start new worker
			payload := map[string]interface{}{
				"token":      route.Token,
				"project_id": route.ProjectId,
				"code_name":  route.CodeName,
			}
			workerapi := worker.New()
			workerapi.Settings.UseConfigMap(payload)
			jsonPayload, err := json.Marshal(payload)
			if err != nil {
				fmt.Println("Couldn't marshal json!", err)
				return
			}
			timeout := time.Second*120
			task := worker.Task{
				CodeName: route.CodeName,
				Payload:  string(jsonPayload),
				Timeout:  &timeout, // let's have these die quickly while testing
			}
			tasks := make([]worker.Task, 1)
			tasks[0] = task
			taskIds, err := workerapi.TaskQueue(tasks...)
			fmt.Println("Tasks queued.", taskIds)
			if err != nil {
				fmt.Println("Couldn't queue up worker!", err)
				return
			}
		}
		// start new worker if it's a connection error
		return
	}
	fmt.Println("Served!")
	// todo: how to handle destination failures. I got this in log output when testing a bad endpoint:
	// 2012/12/26 23:22:08 http: proxy error: dial tcp 127.0.0.1:8082: connection refused
}

// When a worker starts up, it calls this
func AddWorker(w http.ResponseWriter, req *http.Request) {
	log.Println("AddWorker called!")

//	s, err := ioutil.ReadAll(req.Body)
//	fmt.Println("req.body:", err, string(s))

	// get project id and token
	projectId := req.FormValue("project_id")
	token := req.FormValue("token")
	codeName := req.FormValue("code_name")
	fmt.Println("project_id:", projectId, "token:", token, "code_name:", codeName)

	// check header for what operation to perform
	routerHeader := req.Header.Get("Iron-Router")
	if routerHeader == "register" {
		route := Route{}
		if !common.ReadJSON(w, req, &route) {
			return
		}
		fmt.Println("body read into route:", route)
		route.ProjectId = projectId
		route.Token = token
		route.CodeName = codeName
		// todo: do we need to close body?
		putRoute(route)
		fmt.Println("registered route:", route)
		fmt.Fprintln(w, "Host registered successfully.")

	} else {
		r2 := Route2{}
		if !common.ReadJSON(w, req, &r2) {
			return
		}
		// todo: do we need to close body?
		fmt.Println("DECODED:", r2)
		route, err := getRoute(r2.Host)
		//		route := routingTable[r2.Host]
		if err != nil {
			common.SendError(w, 400, fmt.Sprintln("This host is not registered!", err))
			return
			//			route = &Route{}
		}
		fmt.Println("ROUTE:", route)
		route.Destinations = append(route.Destinations, r2.Dest)
		fmt.Println("ROUTE new:", route)
		putRoute(route)
		//		routingTable[r2.Host] = route
		//		fmt.Println("New routing table:", routingTable)
		fmt.Fprintln(w, "Worker added")
	}
}

func getRoute(host string) (Route, error) {
	rx, err := icache.Get(host)
	route := Route{}
	if err == nil {
		route = rx.(Route)
	}
	return route, err
}

func putRoute(route Route) {
	item := cache.Item{}
	item.Value = route
	icache.Put(route.Host, &item)
}
<commit_msg>logging errors on putroute.<commit_after>/*

For keeping a minimum running, perhaps when doing a routing table update, if destination hosts are all
 expired or about to expire we start more.

*/

package main

import (
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/iron-io/iron_go/cache"
	"github.com/iron-io/iron_go/worker"
	"github.com/iron-io/common"
	"log"
	"math/rand"
	// "net"
	"net/http"
	"net/url"
	"reflect"
	"strings"
	"time"
	"runtime"
	"flag"
//	"io/ioutil"
)

var config struct {
	Iron struct {
	Token      string `json:"token"`
	ProjectId  string `json:"project_id"`
} `json:"iron"`
	Logging       struct {
	To     string `json:"to"`
	Level  string `json:"level"`
	Prefix string `json:"prefix"`
}
}

//var routingTable = map[string]*Route{}
var icache = cache.New("routing-table")

func init() {

}

type Route struct {
	// TODO: Change destinations to a simple cache so it can expire entries after 55 minutes (the one we use in common?)
	Host         string `json:"host"`
	Destinations []string  `json:"destinations"`
	ProjectId    string  `json:"project_id"`
	Token        string  `json:"token"` // store this so we can queue up new workers on demand
	CodeName     string  `json:"code_name"`
}

// for adding new hosts
type Route2 struct {
	Host string `json:"host"`
	Dest string `json:"dest"`
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	log.Println("Running on", runtime.NumCPU(), "CPUs")

	var configFile string
	var env string
	flag.StringVar(&configFile, "c", "", "Config file name")
	// when this was e, it was erroring out.
	flag.StringVar(&env, "e2", "development", "environment")

	flag.Parse() // Scans the arg list and sets up flags

	// Deployer is now passing -c in since we're using upstart and it doesn't know what directory to run in
	if configFile == "" {
		configFile = "config_" + env + ".json"
	}

	common.LoadConfig("iron_mq", configFile, &config)
	common.SetLogLevel(config.Logging.Level)
	common.SetLogLocation(config.Logging.To, config.Logging.Prefix)

	icache.Settings.UseConfigMap(map[string]interface{}{"token": config.Iron.Token, "project_id": config.Iron.ProjectId})

	r := mux.NewRouter()
	s := r.Headers("Iron-Router", "").Subrouter()
	s.HandleFunc("/", AddWorker)
	r.HandleFunc("/addworker", AddWorker)

	r.HandleFunc("/", ProxyFunc)

	http.Handle("/", r)
	port := 80
	fmt.Println("listening and serving on port", port)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%v", port), nil))
}

func ProxyFunc(w http.ResponseWriter, req *http.Request) {
	fmt.Println("HOST:", req.Host)
	host := strings.Split(req.Host, ":")[0]

	// We look up the destinations in the routing table and there can be 3 possible scenarios:
	// 1) This host was never registered so we return 404
	// 2) This host has active workers so we do the proxy
	// 3) This host has no active workers so we queue one (or more) up and return a 503 or something with message that says "try again in a minute"
	//	route := routingTable[host]
	route, err := getRoute(host)
	// choose random dest
	if err != nil {
		common.SendError(w, 400, fmt.Sprintln("Host not configured or error!", err))
		return
	}
	//	if route == nil { // route.Host == "" {
	//		common.SendError(w, 400, fmt.Sprintln(w, "Host not configured!"))
	//		return
	//	}
	destIndex := rand.Intn(len(route.Destinations))
	destUrlString := route.Destinations[destIndex]
	// todo: should check if http:// already exists.
	destUrlString2 := "http://" + destUrlString
	destUrl, err := url.Parse(destUrlString2)
	if err != nil {
		fmt.Println("error!", err)
		panic(err)
	}
	fmt.Println("proxying to", destUrl)
	proxy := NewSingleHostReverseProxy(destUrl)
	err = proxy.ServeHTTP(w, req)
	if err != nil {
		fmt.Println("Error proxying!", err)
		etype := reflect.TypeOf(err)
		fmt.Println("err type:", etype)
		w.WriteHeader(http.StatusInternalServerError)
		// can't figure out how to compare types so comparing strings.... lame.
		if strings.Contains(etype.String(), "net.OpError") { // == reflect.TypeOf(net.OpError{}) { // couldn't figure out a better way to do this
			if len(route.Destinations) > 1 {
				fmt.Println("It's a network error, removing this destination from routing table.")
				route.Destinations = append(route.Destinations[:destIndex], route.Destinations[destIndex + 1:]...)
				err := putRoute(route)
				if err != nil {
					fmt.Println("couldn't update routing table 1", err)
					common.SendError(w, 400, fmt.Sprintln("couldn't update routing table 1", err))
					return
				}
				fmt.Println("New route:", route)
				return
			} else {
				fmt.Println("It's a network error and no other destinations available so we're going to remove it and start new task.")
				route.Destinations = append(route.Destinations[:destIndex], route.Destinations[destIndex + 1:]...)
				err := putRoute(route)
				if err != nil {
					fmt.Println("couldn't update routing table:", err)
					common.SendError(w, 400, fmt.Sprintln("couldn't update routing table", err))
					return
				}
				fmt.Println("New route:", route)
			}
			// start new worker
			payload := map[string]interface{}{
				"token":      route.Token,
				"project_id": route.ProjectId,
				"code_name":  route.CodeName,
			}
			workerapi := worker.New()
			workerapi.Settings.UseConfigMap(payload)
			jsonPayload, err := json.Marshal(payload)
			if err != nil {
				fmt.Println("Couldn't marshal json!", err)
				return
			}
			timeout := time.Second*120
			task := worker.Task{
				CodeName: route.CodeName,
				Payload:  string(jsonPayload),
				Timeout:  &timeout, // let's have these die quickly while testing
			}
			tasks := make([]worker.Task, 1)
			tasks[0] = task
			taskIds, err := workerapi.TaskQueue(tasks...)
			fmt.Println("Tasks queued.", taskIds)
			if err != nil {
				fmt.Println("Couldn't queue up worker!", err)
				return
			}
		}
		// start new worker if it's a connection error
		return
	}
	fmt.Println("Served!")
	// todo: how to handle destination failures. I got this in log output when testing a bad endpoint:
	// 2012/12/26 23:22:08 http: proxy error: dial tcp 127.0.0.1:8082: connection refused
}

// When a worker starts up, it calls this
func AddWorker(w http.ResponseWriter, req *http.Request) {
	log.Println("AddWorker called!")

//	s, err := ioutil.ReadAll(req.Body)
//	fmt.Println("req.body:", err, string(s))

	// get project id and token
	projectId := req.FormValue("project_id")
	token := req.FormValue("token")
	codeName := req.FormValue("code_name")
	fmt.Println("project_id:", projectId, "token:", token, "code_name:", codeName)

	// check header for what operation to perform
	routerHeader := req.Header.Get("Iron-Router")
	if routerHeader == "register" {
		route := Route{}
		if !common.ReadJSON(w, req, &route) {
			return
		}
		fmt.Println("body read into route:", route)
		route.ProjectId = projectId
		route.Token = token
		route.CodeName = codeName
		// todo: do we need to close body?
		err := putRoute(route)
		if err != nil {
			fmt.Println("couldn't register host:", err)
			common.SendError(w, 400, fmt.Sprintln("Could not register host!", err))
			return
		}
		fmt.Println("registered route:", route)
		fmt.Fprintln(w, "Host registered successfully.")

	} else {
		r2 := Route2{}
		if !common.ReadJSON(w, req, &r2) {
			return
		}
		// todo: do we need to close body?
		fmt.Println("DECODED:", r2)
		route, err := getRoute(r2.Host)
		//		route := routingTable[r2.Host]
		if err != nil {
			common.SendError(w, 400, fmt.Sprintln("This host is not registered!", err))
			return
			//			route = &Route{}
		}
		fmt.Println("ROUTE:", route)
		route.Destinations = append(route.Destinations, r2.Dest)
		fmt.Println("ROUTE new:", route)
		err = putRoute(route)
		if err != nil {
			fmt.Println("couldn't register host:", err)
			common.SendError(w, 400, fmt.Sprintln("Could not register host!", err))
			return
		}
		//		routingTable[r2.Host] = route
		//		fmt.Println("New routing table:", routingTable)
		fmt.Fprintln(w, "Worker added")
	}
}

func getRoute(host string) (Route, error) {
	rx, err := icache.Get(host)
	route := Route{}
	if err == nil {
		route = rx.(Route)
	}
	return route, err
}

func putRoute(route Route) (error) {
	item := cache.Item{}
	item.Value = route
	err := icache.Put(route.Host, &item)
	return err
}
<|endoftext|>


<commit_before>package maptiles

import (
	"fmt"
	"github.com/fawick/go-mapnik/mapnik"
	"io/ioutil"
	"log"
	"os"
)

type Generator struct {
	MapFile string
	TileDir string
	Threads int
}

func ensureDirExists(path string) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		os.Mkdir(path, 0755)
	}
}

// Generates tile files as a <zoom>/<x>/<y>.png file hierarchy in the current
// work directory.
func (g *Generator) Run(lowLeft, upRight mapnik.Coord, minZ, maxZ uint64, name string) {
	c := make(chan TileCoord)
	q := make(chan bool)

	log.Println("starting job", name)

	ensureDirExists(g.TileDir)

	for i := 0; i < g.Threads; i++ {
		go func(id int, ctc <-chan TileCoord, q chan bool) {
			requests := NewTileRendererChan(g.MapFile)
			results := make(chan TileFetchResult)
			for t := range ctc {
				requests <- TileFetchRequest{t, results}
				r := <-results
				ioutil.WriteFile(r.Coord.OSMFilename(), r.BlobPNG, 0644)
			}
			q <- true
		}(i, c, q)
	}

	ll0 := [2]float64{lowLeft.X, upRight.Y}
	ll1 := [2]float64{upRight.X, lowLeft.Y}

	for z := minZ; z <= maxZ; z++ {
		px0 := fromLLtoPixel(ll0, z)
		px1 := fromLLtoPixel(ll1, z)

		ensureDirExists(fmt.Sprintf("%d", z))
		for x := uint64(px0[0] / 256.0); x <= uint64(px1[0]/256.0); x++ {
			ensureDirExists(fmt.Sprintf("%d/%d", z, x))
			for y := uint64(px0[1] / 256.0); y <= uint64(px1[1]/256.0); y++ {
				c <- TileCoord{x, y, z, false, ""}
			}
		}
	}
	close(c)
	for i := 0; i < g.Threads; i++ {
		<-q
	}
}
<commit_msg>tile server logging<commit_after>package maptiles

import (
	"fmt"
	"github.com/sjsafranek/go-mapnik/mapnik"
	"io/ioutil"
	"log"
	"os"
)

type Generator struct {
	MapFile string
	TileDir string
	Threads int
}

func ensureDirExists(path string) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		os.Mkdir(path, 0755)
	}
}

// Generates tile files as a <zoom>/<x>/<y>.png file hierarchy in the current
// work directory.
func (g *Generator) Run(lowLeft, upRight mapnik.Coord, minZ, maxZ uint64, name string) {
	c := make(chan TileCoord)
	q := make(chan bool)

	log.Println("starting job", name)

	ensureDirExists(g.TileDir)

	for i := 0; i < g.Threads; i++ {
		go func(id int, ctc <-chan TileCoord, q chan bool) {
			requests := NewTileRendererChan(g.MapFile)
			results := make(chan TileFetchResult)
			for t := range ctc {
				requests <- TileFetchRequest{t, results}
				r := <-results
				ioutil.WriteFile(r.Coord.OSMFilename(), r.BlobPNG, 0644)
			}
			q <- true
		}(i, c, q)
	}

	ll0 := [2]float64{lowLeft.X, upRight.Y}
	ll1 := [2]float64{upRight.X, lowLeft.Y}

	for z := minZ; z <= maxZ; z++ {
		px0 := fromLLtoPixel(ll0, z)
		px1 := fromLLtoPixel(ll1, z)

		ensureDirExists(fmt.Sprintf("%d", z))
		for x := uint64(px0[0] / 256.0); x <= uint64(px1[0]/256.0); x++ {
			ensureDirExists(fmt.Sprintf("%d/%d", z, x))
			for y := uint64(px0[1] / 256.0); y <= uint64(px1[1]/256.0); y++ {
				c <- TileCoord{x, y, z, false, ""}
			}
		}
	}
	close(c)
	for i := 0; i < g.Threads; i++ {
		<-q
	}
}
<|endoftext|>


<commit_before>package mackerel

import (
	"encoding/json"
	"fmt"
	"net/http"
)

/*
`/dashboards` Response
Legacy
{
  "dashboards": [
	{
	  "id": "2c5bLca8d",
	  "title": "My Dashboard",
	  "bodyMarkdown": "# A test dashboard",
	  "urlPath": "2u4PP3TJqbu",
	  "createdAt": 1439346145003,
	  "updatedAt": 1439346145003,
	  "isLegacy": true
	}
  ]
}

Current
{
	"dashboards": [
		{
			"id":        "2c5bLca8e",
			"title":     "My Custom Dashboard(Current)",
			"urlPath":   "2u4PP3TJqbv",
			"createdAt": 1552909732,
			"updatedAt": 1552992837,
			"memo":      "A test Current Dashboard"
		}
	]
}
*/

/*
`/dashboards/${ID}` Response`
Legacy
{
	"id": "2c5bLca8d",
	"title": "My Dashboard",
	"bodyMarkdown": "# A test dashboard",
	"urlPath": "2u4PP3TJqbu",
	"createdAt": 1439346145003,
	"updatedAt": 1439346145003,
	"isLegacy": true
}
Current
{
  "id": "2c5bLca8e",
  "createdAt": 1552909732,
  "updatedAt": 1552992837,
  "title": "My Custom Dashboard(Current),
  "urlPath": "2u4PP3TJqbv",
  "memo": "A test Current Dashboard",
  "widgets": [
    {
      "type": "markdown",
      "title": "markdown",
      "markdown": "# body",
      "layout": {
        "x": 0,
        "y": 0,
        "width": 24,
        "height": 3
      }
    },
    {
      "type": "graph",
      "title": "graph",
      "graph": {
        "type": "host",
        "hostId": "2u4PP3TJqbw",
        "name": "loadavg.loadavg15"
      },
      "layout": {
        "x": 0,
        "y": 7,
        "width": 8,
        "height": 10
      }
    },
    {
      "type": "value",
      "title": "value",
      "metric": {
        "type": "expression",
        "expression": "alias(scale(\nsum(\n  group(\n    host(2u4PP3TJqbx,loadavg.*)\n  )\n),\n1\n), 'test')"
      },
      "layout": {
        "x": 0,
        "y": 17,
        "width": 8,
        "height": 5
      }
    }
  ]
}
*/

// Dashboard information
type Dashboard struct {
	ID           string   `json:"id,omitempty"`
	Title        string   `json:"title,omitempty"`
	BodyMarkDown string   `json:"bodyMarkdown,omitempty"`
	URLPath      string   `json:"urlPath,omitempty"`
	CreatedAt    int64    `json:"createdAt,omitempty"`
	UpdatedAt    int64    `json:"updatedAt,omitempty"`
	IsLegacy     bool     `json:"isLegacy,omitempty"`
	Memo         string   `json:"memo,omitempty"`
	Widgets      []Widget `json:"widgets,omitenpty"`
}

// Widget information
type Widget struct {
	Type     string `json:"type,omitempty"`
	Title    string `json:"title,omitempty"`
	Metric   Metric `json:"metric,omitempty"`
	Graph    Graph  `json:"graph,omitempty"`
	Layout   Layout `json:"layout,omitempty"`
	Markdown string `json:"markdown,omitempty"`
	Range    Range  `json:"range,omitempty"`
}

// Metric information
type Metric struct {
	Type        string `json:"type,omitempty"`
	HostID      string `json:"hostId,omitempty"`
	Name        string `json:"name,omitempty"`
	ServiceName string `json:"serviceName,omitempty"`
	Expression  string `json:"expression,omitempty"`
}

// Graph information
type Graph struct {
	Type         string `json:"type,omitempty"`
	HostID       string `json:"hostId,omitempty"`
	Name         string `json:"name,omitempty"`
	RoleFullName string `json:"roleFullname,omitempty"`
	ServiceName  string `json:"serviceName,omitempty"`
	Expression   string `json:"expression,omitempty"`
	IsStacked    bool   `json:"isStacked,omitempty"`
}

// Range information
type Range struct {
	Type   string `json:"type,omitempty"`
	Period int64  `json:"period,omitempty"`
	Offset int64  `json:"offset,omitempty"`
	Start  int64  `json:"start,omitempty"`
	End    int64  `json:"end,omitempty"`
}

// Layout information
type Layout struct {
	X      int64 `json:"x,omitempty"`
	Y      int64 `json:"y,omitempty"`
	Width  int64 `json:"width,omitempty"`
	Height int64 `json:"height,omitempty"`
}

// FindDashboards find dashboards
func (c *Client) FindDashboards() ([]*Dashboard, error) {
	req, err := http.NewRequest("GET", c.urlFor("/api/v0/dashboards").String(), nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.Request(req)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var data struct {
		Dashboards []*Dashboard `json:"dashboards"`
	}
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		return nil, err
	}

	return data.Dashboards, err
}

// FindDashboard find dashboard
func (c *Client) FindDashboard(dashboardID string) (*Dashboard, error) {
	req, err := http.NewRequest("GET", c.urlFor(fmt.Sprintf("/api/v0/dashboards/%s", dashboardID)).String(), nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.Request(req)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var data Dashboard
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		return nil, err
	}
	return &data, err
}

// CreateDashboard creating dashboard
func (c *Client) CreateDashboard(param *Dashboard) (*Dashboard, error) {
	resp, err := c.PostJSON("/api/v0/dashboards", param)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var data Dashboard
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		return nil, err
	}
	return &data, nil
}

// UpdateDashboard update dashboard
func (c *Client) UpdateDashboard(dashboardID string, param *Dashboard) (*Dashboard, error) {
	resp, err := c.PutJSON(fmt.Sprintf("/api/v0/dashboards/%s", dashboardID), param)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var data Dashboard
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		return nil, err
	}
	return &data, nil
}

// DeleteDashboard delete dashboard
func (c *Client) DeleteDashboard(dashboardID string) (*Dashboard, error) {
	req, err := http.NewRequest(
		"DELETE",
		c.urlFor(fmt.Sprintf("/api/v0/dashboards/%s", dashboardID)).String(),
		nil,
	)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Content-Type", "application/json")

	resp, err := c.Request(req)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var data Dashboard
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		return nil, err
	}
	return &data, nil
}
<commit_msg>update sample's indent<commit_after>package mackerel

import (
	"encoding/json"
	"fmt"
	"net/http"
)

/*
`/dashboards` Response
Legacy
{
  "dashboards": [
    {
       "id": "2c5bLca8d",
       "title": "My Dashboard",
       "bodyMarkdown": "# A test dashboard",
       "urlPath": "2u4PP3TJqbu",
       "createdAt": 1439346145003,
       "updatedAt": 1439346145003,
       "isLegacy": true
    }
  ]
}

Current
{
  "dashboards": [
    {
      "id":        "2c5bLca8e",
      "title":     "My Custom Dashboard(Current)",
      "urlPath":   "2u4PP3TJqbv",
      "createdAt": 1552909732,
      "updatedAt": 1552992837,
      "memo":      "A test Current Dashboard"
    }
  ]
}
*/

/*
`/dashboards/${ID}` Response`
Legacy
{
  "id": "2c5bLca8d",
  "title": "My Dashboard",
  "bodyMarkdown": "# A test dashboard",
  "urlPath": "2u4PP3TJqbu",
  "createdAt": 1439346145003,
  "updatedAt": 1439346145003,
  "isLegacy": true
}
Current
{
  "id": "2c5bLca8e",
  "createdAt": 1552909732,
  "updatedAt": 1552992837,
  "title": "My Custom Dashboard(Current),
  "urlPath": "2u4PP3TJqbv",
  "memo": "A test Current Dashboard",
  "widgets": [
    {
      "type": "markdown",
      "title": "markdown",
      "markdown": "# body",
      "layout": {
        "x": 0,
        "y": 0,
        "width": 24,
        "height": 3
      }
    },
    {
      "type": "graph",
      "title": "graph",
      "graph": {
        "type": "host",
        "hostId": "2u4PP3TJqbw",
        "name": "loadavg.loadavg15"
      },
      "layout": {
        "x": 0,
        "y": 7,
        "width": 8,
        "height": 10
      }
    },
    {
      "type": "value",
      "title": "value",
      "metric": {
        "type": "expression",
        "expression": "alias(scale(\nsum(\n  group(\n    host(2u4PP3TJqbx,loadavg.*)\n  )\n),\n1\n), 'test')"
      },
      "layout": {
        "x": 0,
        "y": 17,
        "width": 8,
        "height": 5
      }
    }
  ]
}
*/

// Dashboard information
type Dashboard struct {
	ID           string   `json:"id,omitempty"`
	Title        string   `json:"title,omitempty"`
	BodyMarkDown string   `json:"bodyMarkdown,omitempty"`
	URLPath      string   `json:"urlPath,omitempty"`
	CreatedAt    int64    `json:"createdAt,omitempty"`
	UpdatedAt    int64    `json:"updatedAt,omitempty"`
	IsLegacy     bool     `json:"isLegacy,omitempty"`
	Memo         string   `json:"memo,omitempty"`
	Widgets      []Widget `json:"widgets,omitenpty"`
}

// Widget information
type Widget struct {
	Type     string `json:"type,omitempty"`
	Title    string `json:"title,omitempty"`
	Metric   Metric `json:"metric,omitempty"`
	Graph    Graph  `json:"graph,omitempty"`
	Layout   Layout `json:"layout,omitempty"`
	Markdown string `json:"markdown,omitempty"`
	Range    Range  `json:"range,omitempty"`
}

// Metric information
type Metric struct {
	Type        string `json:"type,omitempty"`
	HostID      string `json:"hostId,omitempty"`
	Name        string `json:"name,omitempty"`
	ServiceName string `json:"serviceName,omitempty"`
	Expression  string `json:"expression,omitempty"`
}

// Graph information
type Graph struct {
	Type         string `json:"type,omitempty"`
	HostID       string `json:"hostId,omitempty"`
	Name         string `json:"name,omitempty"`
	RoleFullName string `json:"roleFullname,omitempty"`
	ServiceName  string `json:"serviceName,omitempty"`
	Expression   string `json:"expression,omitempty"`
	IsStacked    bool   `json:"isStacked,omitempty"`
}

// Range information
type Range struct {
	Type   string `json:"type,omitempty"`
	Period int64  `json:"period,omitempty"`
	Offset int64  `json:"offset,omitempty"`
	Start  int64  `json:"start,omitempty"`
	End    int64  `json:"end,omitempty"`
}

// Layout information
type Layout struct {
	X      int64 `json:"x,omitempty"`
	Y      int64 `json:"y,omitempty"`
	Width  int64 `json:"width,omitempty"`
	Height int64 `json:"height,omitempty"`
}

// FindDashboards find dashboards
func (c *Client) FindDashboards() ([]*Dashboard, error) {
	req, err := http.NewRequest("GET", c.urlFor("/api/v0/dashboards").String(), nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.Request(req)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var data struct {
		Dashboards []*Dashboard `json:"dashboards"`
	}
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		return nil, err
	}

	return data.Dashboards, err
}

// FindDashboard find dashboard
func (c *Client) FindDashboard(dashboardID string) (*Dashboard, error) {
	req, err := http.NewRequest("GET", c.urlFor(fmt.Sprintf("/api/v0/dashboards/%s", dashboardID)).String(), nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.Request(req)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var data Dashboard
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		return nil, err
	}
	return &data, err
}

// CreateDashboard creating dashboard
func (c *Client) CreateDashboard(param *Dashboard) (*Dashboard, error) {
	resp, err := c.PostJSON("/api/v0/dashboards", param)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var data Dashboard
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		return nil, err
	}
	return &data, nil
}

// UpdateDashboard update dashboard
func (c *Client) UpdateDashboard(dashboardID string, param *Dashboard) (*Dashboard, error) {
	resp, err := c.PutJSON(fmt.Sprintf("/api/v0/dashboards/%s", dashboardID), param)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var data Dashboard
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		return nil, err
	}
	return &data, nil
}

// DeleteDashboard delete dashboard
func (c *Client) DeleteDashboard(dashboardID string) (*Dashboard, error) {
	req, err := http.NewRequest(
		"DELETE",
		c.urlFor(fmt.Sprintf("/api/v0/dashboards/%s", dashboardID)).String(),
		nil,
	)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Content-Type", "application/json")

	resp, err := c.Request(req)
	defer closeResponse(resp)
	if err != nil {
		return nil, err
	}

	var data Dashboard
	err = json.NewDecoder(resp.Body).Decode(&data)
	if err != nil {
		return nil, err
	}
	return &data, nil
}
<|endoftext|>