<commit_before>// The googlecompute package contains a packer.Builder implementation that
// builds images for Google Compute Engine.
package googlecompute

import (
	"context"
	"fmt"
	"log"

	"github.com/hashicorp/hcl/v2/hcldec"
	"github.com/hashicorp/packer/common"
	"github.com/hashicorp/packer/helper/communicator"
	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/packer"
)

// The unique ID for this builder.
const BuilderId = "packer.googlecompute"

// Builder represents a Packer Builder.
type Builder struct {
	config Config
	runner multistep.Runner
}

func (b *Builder) ConfigSpec() hcldec.ObjectSpec { return b.config.FlatMapstructure().HCL2Spec() }

func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) {
	warnings, errs := b.config.Prepare(raws...)
	if errs != nil {
		return nil, warnings, errs
	}
	return nil, warnings, nil
}

// Run executes a googlecompute Packer build and returns a packer.Artifact
// representing a GCE machine image.
func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) {
	cfg := GCEDriverConfig{
		Ui:                            ui,
		ProjectId:                     b.config.ProjectId,
		Account:                       b.config.account,
		ImpersonateServiceAccountName: b.config.ImpersonateServiceAccount,
		VaultOauthEngineName:          b.config.VaultGCPOauthEngine,
	}

	driver, err := NewDriverGCE(cfg)
	if err != nil {
		return nil, err
	}

	// Set up the state.
	state := new(multistep.BasicStateBag)
	state.Put("config", &b.config)
	state.Put("driver", driver)
	state.Put("hook", hook)
	state.Put("ui", ui)

	// Build the steps.
	steps := []multistep.Step{
		new(StepCheckExistingImage),
		&communicator.StepSSHKeyGen{
			CommConf:            &b.config.Comm,
			SSHTemporaryKeyPair: b.config.Comm.SSH.SSHTemporaryKeyPair,
		},
		multistep.If(b.config.PackerDebug,
			&communicator.StepDumpSSHKey{
				Path: fmt.Sprintf("gce_%s.pem", b.config.PackerBuildName),
			},
		),
		&StepImportOSLoginSSHKey{
			Debug: b.config.PackerDebug,
		},
		&StepCreateInstance{
			Debug: b.config.PackerDebug,
		},
		&StepCreateWindowsPassword{
			Debug:        b.config.PackerDebug,
			DebugKeyPath: fmt.Sprintf("gce_windows_%s.pem", b.config.PackerBuildName),
		},
		&StepInstanceInfo{
			Debug: b.config.PackerDebug,
		},
		&StepStartTunnel{
			IAPConf:            &b.config.IAPConfig,
			CommConf:           &b.config.Comm,
			AccountFile:        b.config.AccountFile,
			ImpersonateAccount: b.config.ImpersonateServiceAccount,
			ProjectId:          b.config.ProjectId,
		},
		&communicator.StepConnect{
			Config:      &b.config.Comm,
			Host:        communicator.CommHost(b.config.Comm.Host(), "instance_ip"),
			SSHConfig:   b.config.Comm.SSHConfigFunc(),
			WinRMConfig: winrmConfig,
		},
		new(common.StepProvision),
		&common.StepCleanupTempKeys{
			Comm: &b.config.Comm,
		},
	}
	if _, exists := b.config.Metadata[StartupScriptKey]; exists || b.config.StartupScriptFile != "" {
		steps = append(steps, new(StepWaitStartupScript))
	}
	steps = append(steps, new(StepTeardownInstance), new(StepCreateImage))

	// Run the steps.
	b.runner = common.NewRunner(steps, b.config.PackerConfig, ui)
	b.runner.Run(ctx, state)

	// Report any errors.
	if rawErr, ok := state.GetOk("error"); ok {
		return nil, rawErr.(error)
	}
	if _, ok := state.GetOk("image"); !ok {
		log.Println("Failed to find image in state. Bug?")
		return nil, nil
	}

	artifact := &Artifact{
		image:     state.Get("image").(*Image),
		driver:    driver,
		config:    &b.config,
		StateData: map[string]interface{}{"generated_data": state.Get("generated_data")},
	}
	return artifact, nil
}

// Cancel.
<commit_msg>debug mode: don't dump the ssh key if its from a file<commit_after>// The googlecompute package contains a packer.Builder implementation that
// builds images for Google Compute Engine.
package googlecompute

import (
	"context"
	"fmt"
	"log"

	"github.com/hashicorp/hcl/v2/hcldec"
	"github.com/hashicorp/packer/common"
	"github.com/hashicorp/packer/helper/communicator"
	"github.com/hashicorp/packer/helper/multistep"
	"github.com/hashicorp/packer/packer"
)

// The unique ID for this builder.
const BuilderId = "packer.googlecompute"

// Builder represents a Packer Builder.
type Builder struct {
	config Config
	runner multistep.Runner
}

func (b *Builder) ConfigSpec() hcldec.ObjectSpec { return b.config.FlatMapstructure().HCL2Spec() }

func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) {
	warnings, errs := b.config.Prepare(raws...)
	if errs != nil {
		return nil, warnings, errs
	}
	return nil, warnings, nil
}

// Run executes a googlecompute Packer build and returns a packer.Artifact
// representing a GCE machine image.
func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) {
	cfg := GCEDriverConfig{
		Ui:                            ui,
		ProjectId:                     b.config.ProjectId,
		Account:                       b.config.account,
		ImpersonateServiceAccountName: b.config.ImpersonateServiceAccount,
		VaultOauthEngineName:          b.config.VaultGCPOauthEngine,
	}

	driver, err := NewDriverGCE(cfg)
	if err != nil {
		return nil, err
	}

	// Set up the state.
	state := new(multistep.BasicStateBag)
	state.Put("config", &b.config)
	state.Put("driver", driver)
	state.Put("hook", hook)
	state.Put("ui", ui)

	// Build the steps.
	steps := []multistep.Step{
		new(StepCheckExistingImage),
		&communicator.StepSSHKeyGen{
			CommConf:            &b.config.Comm,
			SSHTemporaryKeyPair: b.config.Comm.SSH.SSHTemporaryKeyPair,
		},
		multistep.If(b.config.PackerDebug && b.config.Comm.SSHPrivateKeyFile == "",
			&communicator.StepDumpSSHKey{
				Path: fmt.Sprintf("gce_%s.pem", b.config.PackerBuildName),
			},
		),
		&StepImportOSLoginSSHKey{
			Debug: b.config.PackerDebug,
		},
		&StepCreateInstance{
			Debug: b.config.PackerDebug,
		},
		&StepCreateWindowsPassword{
			Debug:        b.config.PackerDebug,
			DebugKeyPath: fmt.Sprintf("gce_windows_%s.pem", b.config.PackerBuildName),
		},
		&StepInstanceInfo{
			Debug: b.config.PackerDebug,
		},
		&StepStartTunnel{
			IAPConf:            &b.config.IAPConfig,
			CommConf:           &b.config.Comm,
			AccountFile:        b.config.AccountFile,
			ImpersonateAccount: b.config.ImpersonateServiceAccount,
			ProjectId:          b.config.ProjectId,
		},
		&communicator.StepConnect{
			Config:      &b.config.Comm,
			Host:        communicator.CommHost(b.config.Comm.Host(), "instance_ip"),
			SSHConfig:   b.config.Comm.SSHConfigFunc(),
			WinRMConfig: winrmConfig,
		},
		new(common.StepProvision),
		&common.StepCleanupTempKeys{
			Comm: &b.config.Comm,
		},
	}
	if _, exists := b.config.Metadata[StartupScriptKey]; exists || b.config.StartupScriptFile != "" {
		steps = append(steps, new(StepWaitStartupScript))
	}
	steps = append(steps, new(StepTeardownInstance), new(StepCreateImage))

	// Run the steps.
	b.runner = common.NewRunner(steps, b.config.PackerConfig, ui)
	b.runner.Run(ctx, state)

	// Report any errors.
	if rawErr, ok := state.GetOk("error"); ok {
		return nil, rawErr.(error)
	}
	if _, ok := state.GetOk("image"); !ok {
		log.Println("Failed to find image in state. Bug?")
		return nil, nil
	}

	artifact := &Artifact{
		image:     state.Get("image").(*Image),
		driver:    driver,
		config:    &b.config,
		StateData: map[string]interface{}{"generated_data": state.Get("generated_data")},
	}
	return artifact, nil
}

// Cancel.
<|endoftext|>


<commit_before>package chunkedvec

import "github.com/clipperhouse/typewriter"

var templates = typewriter.TemplateSlice{
	chunkedVec,
}

var chunkedVec = &typewriter.Template{
	Name: "ChunkedVec",
	Text: `
type {{.Name}}ChunkedVec struct {
	List      *list.List
	ChunkSize uint
	Empty     {{.Pointer}}{{.Name}}
}

// Creates a new {{.Name}}ChunkedVec with chunkSize as provided
func New{{.Name}}ChunkedVec(chunkSize uint) *{{.Name}}ChunkedVec {
	if chunkSize == 0 {
		chunkSize = 1024
	}

	return &{{.Name}}ChunkedVec{
		List:      list.New(),
		ChunkSize: chunkSize,
	}
}

// Adds the element to the ChunkedVec and returns the position it was added to
func (cv *{{.Name}}ChunkedVec) Add(element {{.Pointer}}{{.Name}}) (uint, uint) {
	listIndex := 0
	for e := cv.List.Front(); e != nil; e = e.Next() {
		for index, value := range e.Value.([]{{.Pointer}}{{.Name}}) {
			if value == cv.Empty {
				e.Value.([]{{.Pointer}}{{.Name}})[index] = element
				return uint(listIndex), uint(index)
			}
		}

		listIndex++
	}

	slice := make([]{{.Pointer}}{{.Name}}, cv.ChunkSize)
	slice[0] = element
	cv.List.PushBack(slice)

	return uint(listIndex), uint(0)
}

// Overwrites the given position to hold the given value
func (cv *{{.Name}}ChunkedVec) PutAt(element {{.Pointer}}{{.Name}}, listIndex, sliceIndex uint) {
	var i uint = 0
	e := cv.List.Front()
	for ; i < listIndex; e = e.Next() {
		i++
	}

	e.Value.([]{{.Pointer}}{{.Name}})[sliceIndex] = element
}

// Puts the cv.Empty value at the given position
func (cv *{{.Name}}ChunkedVec) DeleteAt(listIndex, sliceIndex uint) {
	cv.PutAt(cv.Empty, listIndex, sliceIndex)
}

// Returns the value that is on the given position
func (cv *{{.Name}}ChunkedVec) Get(listIndex, sliceIndex uint) {{.Pointer}}{{.Name}} {
	var i uint = 0
	e := cv.List.Front()
	for ; i < listIndex; e = e.Next() {
		i++
	}

	return e.Value.([]{{.Pointer}}{{.Name}})[sliceIndex]
}

// Remove list nodes that has arrays that are with the Empty element only
func (cv *{{.Name}}ChunkedVec) Shrink() {
	for e := cv.List.Front(); e != nil; e = e.Next() {
		allEmpty := true
		for _, value := range e.Value.([]{{.Pointer}}{{.Name}}) {
			if value != cv.Empty {
				allEmpty = false
				break
			}
		}

		if allEmpty {
			cv.List.Remove(e)
		}
	}
}

// Returns the number of non-empty valued elements
func (cv *{{.Name}}ChunkedVec) Len() int {
	number := 0

	for e := cv.List.Front(); e != nil; e = e.Next() {
		for _, value := range e.Value.([]{{.Pointer}}{{.Name}}) {
			if value != cv.Empty {
				number++
			}
		}
	}

	return number
}

// Returns the current capacity of the {{.Name}}ChunkedVec
// i.e. the number of elements it can currently hold without growing
func (cv *{{.Name}}ChunkedVec) Cap() int {
	return cv.List.Len() * int(cv.ChunkSize)
}

// Iter returns a channel of type {{.Pointer}}{{.Name}} that you can range over.
func (cv *{{.Name}}ChunkedVec) Iter() <-chan {{.Pointer}}{{.Name}} {
	ch := make(chan {{.Pointer}}{{.Name}})

	go func() {
		for e := cv.List.Front(); e != nil; e = e.Next() {
			for _, value := range e.Value.([]{{.Pointer}}{{.Name}}) {
				ch <- value
			}
			close(ch)
		}
	}()

	return ch
}

// Checks if the {{.Name}}ChunkedVec contains the given element
func (cv *{{.Name}}ChunkedVec) Contains(element {{.Pointer}}{{.Name}}) bool {
	for e := cv.List.Front(); e != nil; e = e.Next() {
		for _, value := range e.Value.([]{{.Pointer}}{{.Name}}) {
			if value == element {
				return true
			}
		}
	}

	return false
}

// Checks if the {{.Name}}ChunkedVec contains all of the given element
func (cv *{{.Name}}ChunkedVec) ContainsAll(searchingFor ...{{.Pointer}}{{.Name}}) bool {
	for _, s := range searchingFor {
		if !cv.Contains(s) {
			return false
		}
	}

	return true
}

// Checks if this {{.Name}}ChunkedVec is equal to another one
// Two {{.Name}}ChunkedVecs are equal if they have the same number of lists
// with slices that have the same values
func (cv *{{.Name}}ChunkedVec) Equal(other *{{.Name}}ChunkedVec) bool {
	// no worries, the complexity of this is O(1)
	if cv.List.Len() != other.List.Len() {
		return false
	}

	e2 := other.List.Front()
	for e1 := cv.List.Front(); e1 != nil; e1 = e1.Next() {
		len1 := len(e1.Value.([]{{.Pointer}}{{.Name}}))
		len2 := len(e2.Value.([]{{.Pointer}}{{.Name}}))
		if len1 != len2 {
			return false
		}

		for i := 0; i < len1; i++ {
			if e1.Value.([]{{.Pointer}}{{.Name}})[i] != e2.Value.([]{{.Pointer}}{{.Name}})[i] {
				return false
			}
		}

		e2 = e2.Next()
	}

	return true
}

// Clone returns a clone of the {{.Name}}ChunkedVec.
// Does NOT clone the underlying elements.
func (cv *{{.Name}}ChunkedVec) Clone() *{{.Name}}ChunkedVec {
	cloned{{.Name}}ChunkedVec := New{{.Name}}ChunkedVec(cv.ChunkSize)

	var listIndex uint = 0
	for e := cv.List.Front(); e != nil; e = e.Next() {
		for index, value := range e.Value.([]{{.Pointer}}{{.Name}}) {
			cloned{{.Name}}ChunkedVec.PutAt(value, listIndex, uint(index))
		}

		listIndex++
	}

	return cloned{{.Name}}ChunkedVec
}

// Clears all the data in the {{.Name}}ChunkedVec
func (cv *{{.Name}}ChunkedVec) Clear() {
	for e := cv.List.Front(); e != nil; e = e.Next() {
		cv.List.Remove(e)
	}
}

func (cv *{{.Name}}ChunkedVec) String() string {
	var buff bytes.Buffer
	fmt.Fprintf(&buff, "{\n")
	for e := cv.List.Front(); e != nil; e = e.Next() {
		slice := e.Value.([]{{.Pointer}}{{.Name}})
		if _, err := fmt.Fprintf(&buff, fmt.Sprintf("\t%s\n", slice)); err != nil {
			panic("Can't write to buffer")
		}
	}
	fmt.Fprintf(&buff, "\n}")

	return buff.String()
}
`,
	TypeConstraint: typewriter.Constraint{Comparable: true},
}
<commit_msg>Add Grow method<commit_after>package chunkedvec

import "github.com/clipperhouse/typewriter"

var templates = typewriter.TemplateSlice{
	chunkedVec,
}

var chunkedVec = &typewriter.Template{
	Name: "ChunkedVec",
	Text: `
type {{.Name}}ChunkedVec struct {
	List      *list.List
	ChunkSize uint
	Empty     {{.Pointer}}{{.Name}}
}

// Creates a new {{.Name}}ChunkedVec with chunkSize as provided
func New{{.Name}}ChunkedVec(chunkSize uint) *{{.Name}}ChunkedVec {
	if chunkSize == 0 {
		chunkSize = 1024
	}

	return &{{.Name}}ChunkedVec{
		List:      list.New(),
		ChunkSize: chunkSize,
	}
}

// Adds the element to the ChunkedVec and returns the position it was added to
func (cv *{{.Name}}ChunkedVec) Add(element {{.Pointer}}{{.Name}}) (uint, uint) {
	listIndex := 0
	for e := cv.List.Front(); e != nil; e = e.Next() {
		for index, value := range e.Value.([]{{.Pointer}}{{.Name}}) {
			if value == cv.Empty {
				e.Value.([]{{.Pointer}}{{.Name}})[index] = element
				return uint(listIndex), uint(index)
			}
		}

		listIndex++
	}

	slice := make([]{{.Pointer}}{{.Name}}, cv.ChunkSize)
	slice[0] = element
	cv.List.PushBack(slice)

	return uint(listIndex), uint(0)
}

// Overwrites the given position to hold the given value
func (cv *{{.Name}}ChunkedVec) PutAt(element {{.Pointer}}{{.Name}}, listIndex, sliceIndex uint) {
	var i uint = 0
	e := cv.List.Front()
	for ; i < listIndex; e = e.Next() {
		i++
	}

	e.Value.([]{{.Pointer}}{{.Name}})[sliceIndex] = element
}

// Puts the cv.Empty value at the given position
func (cv *{{.Name}}ChunkedVec) DeleteAt(listIndex, sliceIndex uint) {
	cv.PutAt(cv.Empty, listIndex, sliceIndex)
}

// Returns the value that is on the given position
func (cv *{{.Name}}ChunkedVec) Get(listIndex, sliceIndex uint) {{.Pointer}}{{.Name}} {
	var i uint = 0
	e := cv.List.Front()
	for ; i < listIndex; e = e.Next() {
		i++
	}

	return e.Value.([]{{.Pointer}}{{.Name}})[sliceIndex]
}

// Adds chunks (list nodes) to the {{.Name}}ChunkedVec
func (cv *{{.Name}}ChunkedVec) Grow(n int) {
	if n < 0 {
		panic("Can't grow {{.Name}}ChunkedVec with a negative amount")
	}

	for i := 0; i < n; i++ {
		slice := make([]{{.Pointer}}{{.Name}}, cv.ChunkSize)
		cv.List.PushBack(slice)
	}
}

// Remove list nodes that has arrays that are with the Empty element only
func (cv *{{.Name}}ChunkedVec) Shrink() {
	for e := cv.List.Front(); e != nil; e = e.Next() {
		allEmpty := true
		for _, value := range e.Value.([]{{.Pointer}}{{.Name}}) {
			if value != cv.Empty {
				allEmpty = false
				break
			}
		}

		if allEmpty {
			cv.List.Remove(e)
		}
	}
}

// Returns the number of non-empty valued elements
func (cv *{{.Name}}ChunkedVec) Len() int {
	number := 0

	for e := cv.List.Front(); e != nil; e = e.Next() {
		for _, value := range e.Value.([]{{.Pointer}}{{.Name}}) {
			if value != cv.Empty {
				number++
			}
		}
	}

	return number
}

// Returns the current capacity of the {{.Name}}ChunkedVec
// i.e. the number of elements it can currently hold without growing
func (cv *{{.Name}}ChunkedVec) Cap() int {
	return cv.List.Len() * int(cv.ChunkSize)
}

// Iter returns a channel of type {{.Pointer}}{{.Name}} that you can range over.
func (cv *{{.Name}}ChunkedVec) Iter() <-chan {{.Pointer}}{{.Name}} {
	ch := make(chan {{.Pointer}}{{.Name}})

	go func() {
		for e := cv.List.Front(); e != nil; e = e.Next() {
			for _, value := range e.Value.([]{{.Pointer}}{{.Name}}) {
				ch <- value
			}
			close(ch)
		}
	}()

	return ch
}

// Checks if the {{.Name}}ChunkedVec contains the given element
func (cv *{{.Name}}ChunkedVec) Contains(element {{.Pointer}}{{.Name}}) bool {
	for e := cv.List.Front(); e != nil; e = e.Next() {
		for _, value := range e.Value.([]{{.Pointer}}{{.Name}}) {
			if value == element {
				return true
			}
		}
	}

	return false
}

// Checks if the {{.Name}}ChunkedVec contains all of the given element
func (cv *{{.Name}}ChunkedVec) ContainsAll(searchingFor ...{{.Pointer}}{{.Name}}) bool {
	for _, s := range searchingFor {
		if !cv.Contains(s) {
			return false
		}
	}

	return true
}

// Checks if this {{.Name}}ChunkedVec is equal to another one
// Two {{.Name}}ChunkedVecs are equal if they have the same number of lists
// with slices that have the same values
func (cv *{{.Name}}ChunkedVec) Equal(other *{{.Name}}ChunkedVec) bool {
	// no worries, the complexity of this is O(1)
	if cv.List.Len() != other.List.Len() {
		return false
	}

	e2 := other.List.Front()
	for e1 := cv.List.Front(); e1 != nil; e1 = e1.Next() {
		len1 := len(e1.Value.([]{{.Pointer}}{{.Name}}))
		len2 := len(e2.Value.([]{{.Pointer}}{{.Name}}))
		if len1 != len2 {
			return false
		}

		for i := 0; i < len1; i++ {
			if e1.Value.([]{{.Pointer}}{{.Name}})[i] != e2.Value.([]{{.Pointer}}{{.Name}})[i] {
				return false
			}
		}

		e2 = e2.Next()
	}

	return true
}

// Clone returns a clone of the {{.Name}}ChunkedVec.
// Does NOT clone the underlying elements.
func (cv *{{.Name}}ChunkedVec) Clone() *{{.Name}}ChunkedVec {
	cloned{{.Name}}ChunkedVec := New{{.Name}}ChunkedVec(cv.ChunkSize)

	var listIndex uint = 0
	for e := cv.List.Front(); e != nil; e = e.Next() {
		for index, value := range e.Value.([]{{.Pointer}}{{.Name}}) {
			cloned{{.Name}}ChunkedVec.PutAt(value, listIndex, uint(index))
		}

		listIndex++
	}

	return cloned{{.Name}}ChunkedVec
}

// Clears all the data in the {{.Name}}ChunkedVec
func (cv *{{.Name}}ChunkedVec) Clear() {
	for e := cv.List.Front(); e != nil; e = e.Next() {
		cv.List.Remove(e)
	}
}

func (cv *{{.Name}}ChunkedVec) String() string {
	var buff bytes.Buffer
	fmt.Fprintf(&buff, "{\n")
	for e := cv.List.Front(); e != nil; e = e.Next() {
		slice := e.Value.([]{{.Pointer}}{{.Name}})
		if _, err := fmt.Fprintf(&buff, fmt.Sprintf("\t%s\n", slice)); err != nil {
			panic("Can't write to buffer")
		}
	}
	fmt.Fprintf(&buff, "\n}")

	return buff.String()
}
`,
	TypeConstraint: typewriter.Constraint{Comparable: true},
}
<|endoftext|>


<commit_before>package statistics

import "testing"

func TestMedian(t *testing.T) {
	median, _ := Median([]float64{1,2,3})
	if median != 2 {
		t.Error("Median must be 2")
	}

	median, _ = Median([]float64{1,2})
	if median != 1.5 {
		t.Error("Median must be 1 / 2")
	}

	median, _ = Median([]float64{1,2,3,4})
	if median != 2.5 {
		t.Error("Median must be equal to 2.5")
	}
}<commit_msg>Test for error cases in median<commit_after>package statistics

import "testing"

func TestMedian(t *testing.T) {
	median, _ := Median([]float64{1,2,3})
	if median != 2 {
		t.Error("Median must be 2")
	}

	median, _ = Median([]float64{1,2})
	if median != 1.5 {
		t.Error("Median must be 1 / 2")
	}

	median, _ = Median([]float64{1,2,3,4})
	if median != 2.5 {
		t.Error("Median must be equal to 2.5")
	}

	median, err := Median([]float64{})
	if err == nil {
		t.Error("Empty list. Must return an error")
	}
}<|endoftext|>


<commit_before>// Package xlog is a logger coupled with HTTP net/context aware middleware.
//
// Features:
//
//   - Per request log context
//   - Per request and/or per message key/value fields
//   - Log levels (Debug, Info, Warn, Error)
//   - Custom output
//   - Automatically gathers request context like User-Agent, IP etc.
//   - Drops message rather than blocking execution
//
// It works best in combination with github.com/rs/xhandler.
package xlog // import "github.com/rs/xlog"

import (
	"fmt"
	"io"
	"strings"
	"time"
)

// Logger is per request logger interface
type Logger interface {
	// Implements io.Writer so it can be set a output of log.Logger
	io.Writer

	// SetField sets a field on the logger's context. All future messages on this logger
	// will have this field set.
	SetField(name string, value interface{})
	// Debug logs a debug message. If last parameter is a map[string]string, it's content
	// is added as fields to the message.
	Debug(v ...interface{})
	// Debug logs a debug message with format. If last parameter is a map[string]string,
	// it's content is added as fields to the message.
	Debugf(format string, v ...interface{})
	// Info logs a info message. If last parameter is a map[string]string, it's content
	// is added as fields to the message.
	Info(v ...interface{})
	// Info logs a info message with format. If last parameter is a map[string]string,
	// it's content is added as fields to the message.
	Infof(format string, v ...interface{})
	// Warn logs a warning message. If last parameter is a map[string]string, it's content
	// is added as fields to the message.
	Warn(v ...interface{})
	// Warn logs a warning message with format. If last parameter is a map[string]string,
	// it's content is added as fields to the message.
	Warnf(format string, v ...interface{})
	// Error logs an error message. If last parameter is a map[string]string, it's content
	// is added as fields to the message.
	Error(v ...interface{})
	// Error logs an error message with format. If last parameter is a map[string]string,
	// it's content is added as fields to the message.
	Errorf(format string, v ...interface{})
}

// F represents a set of log message fields string -> interface{}
type F map[string]interface{}

type logger struct {
	level  Level
	output chan map[string]interface{}
	fields map[string]interface{}
}

// Common key names for log messages
const (
	KeyTime    = "time"
	KeyMessage = "message"
	KeyLevel   = "level"
)

var now = time.Now

func (l *logger) send(level Level, msg string, fields map[string]interface{}) {
	if level < l.level {
		return
	}
	data := map[string]interface{}{
		KeyTime:    now(),
		KeyLevel:   level.String(),
		KeyMessage: msg,
	}
	for k, v := range fields {
		data[k] = v
	}
	for k, v := range l.fields {
		data[k] = v
	}
	select {
	case l.output <- data:
		// Sent with success
	default:
		// Channel is full, message dropped
	}
}

func extractFields(v *[]interface{}) map[string]interface{} {
	if l := len(*v); l > 0 {
		if f, ok := (*v)[l-1].(map[string]interface{}); ok {
			*v = (*v)[:l-1]
			return f
		}
		if f, ok := (*v)[l-1].(F); ok {
			*v = (*v)[:l-1]
			return f
		}
	}
	return nil
}

// SetField implements Logger interface
func (l *logger) SetField(name string, value interface{}) {
	if l.fields == nil {
		l.fields = map[string]interface{}{}
	}
	l.fields[name] = value
}

// Debug implements Logger interface
func (l *logger) Debug(v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelDebug, fmt.Sprint(v...), f)
}

// Debugf implements Logger interface
func (l *logger) Debugf(format string, v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelDebug, fmt.Sprintf(format, v...), f)
}

// Info implements Logger interface
func (l *logger) Info(v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelInfo, fmt.Sprint(v...), f)
}

// Infof implements Logger interface
func (l *logger) Infof(format string, v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelInfo, fmt.Sprintf(format, v...), f)
}

// Warn implements Logger interface
func (l *logger) Warn(v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelWarn, fmt.Sprint(v...), f)
}

// Warnf implements Logger interface
func (l *logger) Warnf(format string, v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelWarn, fmt.Sprintf(format, v...), f)
}

// Error implements Logger interface
func (l *logger) Error(v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelError, fmt.Sprint(v...), f)
}

// Errorf implements Logger interface
func (l *logger) Errorf(format string, v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelError, fmt.Sprintf(format, v...), f)
}

// Write implements io.Writer interface
func (l *logger) Write(p []byte) (int, error) {
	msg := strings.TrimRight(string(p), "\n")
	l.Info(msg)
	return len(p), nil
}
<commit_msg>Add the "file" field to all logs with caller info<commit_after>// Package xlog is a logger coupled with HTTP net/context aware middleware.
//
// Features:
//
//   - Per request log context
//   - Per request and/or per message key/value fields
//   - Log levels (Debug, Info, Warn, Error)
//   - Custom output
//   - Automatically gathers request context like User-Agent, IP etc.
//   - Drops message rather than blocking execution
//
// It works best in combination with github.com/rs/xhandler.
package xlog // import "github.com/rs/xlog"

import (
	"fmt"
	"io"
	"path"
	"runtime"
	"strings"
	"time"
)

// Logger is per request logger interface
type Logger interface {
	// Implements io.Writer so it can be set a output of log.Logger
	io.Writer

	// SetField sets a field on the logger's context. All future messages on this logger
	// will have this field set.
	SetField(name string, value interface{})
	// Debug logs a debug message. If last parameter is a map[string]string, it's content
	// is added as fields to the message.
	Debug(v ...interface{})
	// Debug logs a debug message with format. If last parameter is a map[string]string,
	// it's content is added as fields to the message.
	Debugf(format string, v ...interface{})
	// Info logs a info message. If last parameter is a map[string]string, it's content
	// is added as fields to the message.
	Info(v ...interface{})
	// Info logs a info message with format. If last parameter is a map[string]string,
	// it's content is added as fields to the message.
	Infof(format string, v ...interface{})
	// Warn logs a warning message. If last parameter is a map[string]string, it's content
	// is added as fields to the message.
	Warn(v ...interface{})
	// Warn logs a warning message with format. If last parameter is a map[string]string,
	// it's content is added as fields to the message.
	Warnf(format string, v ...interface{})
	// Error logs an error message. If last parameter is a map[string]string, it's content
	// is added as fields to the message.
	Error(v ...interface{})
	// Error logs an error message with format. If last parameter is a map[string]string,
	// it's content is added as fields to the message.
	Errorf(format string, v ...interface{})
}

// F represents a set of log message fields string -> interface{}
type F map[string]interface{}

type logger struct {
	level  Level
	output chan map[string]interface{}
	fields map[string]interface{}
}

// Common key names for log messages
const (
	KeyTime    = "time"
	KeyMessage = "message"
	KeyLevel   = "level"
	KeyFile    = "file"
)

var now = time.Now

func (l *logger) send(level Level, msg string, fields map[string]interface{}) {
	if level < l.level {
		return
	}
	data := map[string]interface{}{
		KeyTime:    now(),
		KeyLevel:   level.String(),
		KeyMessage: msg,
	}
	if _, file, line, ok := runtime.Caller(2); ok {
		data[KeyFile] = fmt.Sprintf("%s:%d", path.Base(file), line)
	}
	for k, v := range fields {
		data[k] = v
	}
	for k, v := range l.fields {
		data[k] = v
	}
	select {
	case l.output <- data:
		// Sent with success
	default:
		// Channel is full, message dropped
	}
}

func extractFields(v *[]interface{}) map[string]interface{} {
	if l := len(*v); l > 0 {
		if f, ok := (*v)[l-1].(map[string]interface{}); ok {
			*v = (*v)[:l-1]
			return f
		}
		if f, ok := (*v)[l-1].(F); ok {
			*v = (*v)[:l-1]
			return f
		}
	}
	return nil
}

// SetField implements Logger interface
func (l *logger) SetField(name string, value interface{}) {
	if l.fields == nil {
		l.fields = map[string]interface{}{}
	}
	l.fields[name] = value
}

// Debug implements Logger interface
func (l *logger) Debug(v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelDebug, fmt.Sprint(v...), f)
}

// Debugf implements Logger interface
func (l *logger) Debugf(format string, v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelDebug, fmt.Sprintf(format, v...), f)
}

// Info implements Logger interface
func (l *logger) Info(v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelInfo, fmt.Sprint(v...), f)
}

// Infof implements Logger interface
func (l *logger) Infof(format string, v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelInfo, fmt.Sprintf(format, v...), f)
}

// Warn implements Logger interface
func (l *logger) Warn(v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelWarn, fmt.Sprint(v...), f)
}

// Warnf implements Logger interface
func (l *logger) Warnf(format string, v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelWarn, fmt.Sprintf(format, v...), f)
}

// Error implements Logger interface
func (l *logger) Error(v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelError, fmt.Sprint(v...), f)
}

// Errorf implements Logger interface
func (l *logger) Errorf(format string, v ...interface{}) {
	f := extractFields(&v)
	l.send(LevelError, fmt.Sprintf(format, v...), f)
}

// Write implements io.Writer interface
func (l *logger) Write(p []byte) (int, error) {
	msg := strings.TrimRight(string(p), "\n")
	l.Info(msg)
	return len(p), nil
}
<|endoftext|>


<commit_before>package sms

import (
	"encoding/xml"
	"fmt"
	"log"
	"os"

	xco "github.com/mndrix/go-xco"
)

// xmppProcess is the piece which interacts with the XMPP network and
// converts those communications into values which the rest of
// sms-over-xmpp can understand.
type xmppProcess struct {
	// where to connect to the XMPP server
	host string
	port int

	// credentials for XMPP auth
	name   string
	secret string

	// channels for communicating with the Gateway process
	gatewayRx chan<- *xco.Message
	gatewayTx <-chan *xco.Message

	// contacted records whether the local and remote JIDs,
	// respectively, have contacted each other during the life of this
	// process.
	contacted map[xco.Address]map[xco.Address]bool
}

// runXmppComponent creates a goroutine for sending and receiving XMPP
// stanzas.  it returns a channel for monitoring the goroutine's
// health.  if that channel closes, the XMPP process has died.
func (x *xmppProcess) run() <-chan struct{} {
	opts := xco.Options{
		Name:         x.name,
		SharedSecret: x.secret,
		Address:      fmt.Sprintf("%s:%d", x.host, x.port),
		Logger:       log.New(os.Stderr, "", log.LstdFlags),
	}

	healthCh := make(chan struct{})
	go x.loop(opts, healthCh)
	return healthCh
}

func (x *xmppProcess) loop(opts xco.Options, healthCh chan<- struct{}) {
	defer func() { close(healthCh) }()

	c, err := xco.NewComponent(opts)
	if err != nil {
		log.Printf("can't create internal XMPP component: %s", err)
		return
	}

	tx, rx, errx := c.RunAsync()
	for {
		select {
		case st := <-rx:
			switch stanza := st.(type) {
			case *xco.Message:
				log.Printf("Message: %+v", stanza)
				if stanza.Body == "" {
					log.Printf("  ignoring message with empty body")
					break
				}
				local := &stanza.Header.To
				remote := &stanza.Header.From
				if x.isFirstContact(local, remote) {
					x.hadContact(local, remote)
					p := x.requestSubscription(local, remote)
					go func() { tx <- p }()
				}
				go func() { x.gatewayRx <- stanza }()
			case *xco.Iq:
				if stanza.IsDiscoInfo() {
					log.Printf("Disco: %+v", stanza)
					ids, features := x.describeService()
					stanza, err = stanza.DiscoInfoReply(ids, features)
					if err == nil {
						go func() { tx <- stanza }()
					}
				} else {
					log.Printf("Iq: %+v", stanza)
				}
			case *xco.Presence:
				log.Printf("Presence: %+v", stanza)
				switch stanza.Type {
				case "probe":
					stanza = x.presenceAvailable(stanza)
					go func() { tx <- stanza }()
				case "subscribe", "unsubscribe":
					stanzas := x.handleSubscribeUnsubscribe(stanza)
					go func() {
						for _, stanza := range stanzas {
							tx <- stanza
						}
					}()
				}
			case *xml.StartElement:
				log.Printf("Unknown: %+v", stanza)
			default:
				panic(fmt.Sprintf("Unexpected stanza type: %#v", st))
			}
		case stanza := <-x.gatewayTx:
			local := &stanza.Header.From
			remote := &stanza.Header.To
			if x.isFirstContact(local, remote) {
				x.hadContact(local, remote)
				p := x.requestSubscription(local, remote)
				go func() {
					tx <- p
					tx <- stanza
				}()
			} else {
				go func() { tx <- stanza }()
			}
		case err = <-errx:
		}

		if err != nil {
			break
		}
	}

	log.Printf("lost XMPP connection: %s", err)
}

func (x *xmppProcess) describeService() ([]xco.DiscoIdentity, []xco.DiscoFeature) {
	ids := []xco.DiscoIdentity{
		{
			Category: "gateway",
			Type:     "sms",
			Name:     "SMS over XMPP",
		},
	}
	features := []xco.DiscoFeature{
		{
			Var: "urn:xmpp:receipts",
		},
	}
	return ids, features
}

// isFirstContact returns true if the entities local and remote have
// not had any contact during the life of this process.  Contact
// includes both messages and subscriptions.
func (x *xmppProcess) isFirstContact(local, remote *xco.Address) bool {
	if x.contacted == nil {
		return true
	}

	local = local.Bare()
	remotes := x.contacted[*local]
	if remotes == nil {
		return true
	}

	remote = remote.Bare()
	return !remotes[*remote]
}

// hadContact records the fact that local and remote have contacted
// each other.  It could be for the first time or any subsequent time.
func (x *xmppProcess) hadContact(local, remote *xco.Address) {
	local = local.Bare()
	remote = remote.Bare()
	if x.contacted == nil {
		x.contacted = make(map[xco.Address]map[xco.Address]bool)
	}
	remotes := x.contacted[*local]
	if remotes == nil {
		remotes = make(map[xco.Address]bool)
		x.contacted[*local] = remotes
	}
	remotes[*remote] = true
}

func (x *xmppProcess) presenceAvailable(p *xco.Presence) *xco.Presence {
	stanza := &xco.Presence{
		Header: xco.Header{
			From: p.Header.To,
			To:   p.Header.From,
			ID:   NewId(),
		},
	}
	return stanza
}

func (x *xmppProcess) handleSubscribeUnsubscribe(p *xco.Presence) []*xco.Presence {
	// RFC says to use bare JIDs
	p.Header.To.ResourcePart = ""
	p.Header.From.ResourcePart = ""

	stanza := &xco.Presence{
		Header: xco.Header{
			From: p.Header.To,
			To:   p.Header.From,
			ID:   NewId(),
		},
	}
	switch p.Type {
	case "subscribe":
		stanza.Type = "subscribed"
		local := &p.Header.To
		remote := &p.Header.From
		stanzas := []*xco.Presence{
			stanza,
			x.presenceAvailable(p),
		}
		if x.isFirstContact(local, remote) {
			x.hadContact(local, remote)
			stanzas = append(stanzas, x.requestSubscription(local, remote))
		}
		return stanzas
	case "unsubscribe":
		stanza.Type = "unavailable"
		return []*xco.Presence{stanza}
	}

	return nil
}

func (x *xmppProcess) requestSubscription(local, remote *xco.Address) *xco.Presence {
	stanza := &xco.Presence{
		Header: xco.Header{
			From: *local,
			To:   *remote,
			ID:   NewId(),
		},
		Type: "subscribe",
	}
	return stanza
}
<commit_msg>Temporarily stop sending "subscribe" presence<commit_after>package sms

import (
	"encoding/xml"
	"fmt"
	"log"
	"os"

	xco "github.com/mndrix/go-xco"
)

// xmppProcess is the piece which interacts with the XMPP network and
// converts those communications into values which the rest of
// sms-over-xmpp can understand.
type xmppProcess struct {
	// where to connect to the XMPP server
	host string
	port int

	// credentials for XMPP auth
	name   string
	secret string

	// channels for communicating with the Gateway process
	gatewayRx chan<- *xco.Message
	gatewayTx <-chan *xco.Message

	// contacted records whether the local and remote JIDs,
	// respectively, have contacted each other during the life of this
	// process.
	contacted map[xco.Address]map[xco.Address]bool
}

// runXmppComponent creates a goroutine for sending and receiving XMPP
// stanzas.  it returns a channel for monitoring the goroutine's
// health.  if that channel closes, the XMPP process has died.
func (x *xmppProcess) run() <-chan struct{} {
	opts := xco.Options{
		Name:         x.name,
		SharedSecret: x.secret,
		Address:      fmt.Sprintf("%s:%d", x.host, x.port),
		Logger:       log.New(os.Stderr, "", log.LstdFlags),
	}

	healthCh := make(chan struct{})
	go x.loop(opts, healthCh)
	return healthCh
}

func (x *xmppProcess) loop(opts xco.Options, healthCh chan<- struct{}) {
	defer func() { close(healthCh) }()

	c, err := xco.NewComponent(opts)
	if err != nil {
		log.Printf("can't create internal XMPP component: %s", err)
		return
	}

	tx, rx, errx := c.RunAsync()
	for {
		select {
		case st := <-rx:
			switch stanza := st.(type) {
			case *xco.Message:
				log.Printf("Message: %+v", stanza)
				if stanza.Body == "" {
					log.Printf("  ignoring message with empty body")
					break
				}
				local := &stanza.Header.To
				remote := &stanza.Header.From
				if x.isFirstContact(local, remote) {
					x.hadContact(local, remote)
					p := x.requestSubscription(local, remote)
					go func() { tx <- p }()
				}
				go func() { x.gatewayRx <- stanza }()
			case *xco.Iq:
				if stanza.IsDiscoInfo() {
					log.Printf("Disco: %+v", stanza)
					ids, features := x.describeService()
					stanza, err = stanza.DiscoInfoReply(ids, features)
					if err == nil {
						go func() { tx <- stanza }()
					}
				} else {
					log.Printf("Iq: %+v", stanza)
				}
			case *xco.Presence:
				log.Printf("Presence: %+v", stanza)
				switch stanza.Type {
				case "probe":
					stanza = x.presenceAvailable(stanza)
					go func() { tx <- stanza }()
				case "subscribe", "unsubscribe":
					stanzas := x.handleSubscribeUnsubscribe(stanza)
					go func() {
						for _, stanza := range stanzas {
							tx <- stanza
						}
					}()
				}
			case *xml.StartElement:
				log.Printf("Unknown: %+v", stanza)
			default:
				panic(fmt.Sprintf("Unexpected stanza type: %#v", st))
			}
		case stanza := <-x.gatewayTx:
			local := &stanza.Header.From
			remote := &stanza.Header.To
			if x.isFirstContact(local, remote) {
				x.hadContact(local, remote)
				//p := x.requestSubscription(local, remote)
				go func() {
					//tx <- p
					tx <- stanza
				}()
			} else {
				go func() { tx <- stanza }()
			}
		case err = <-errx:
		}

		if err != nil {
			break
		}
	}

	log.Printf("lost XMPP connection: %s", err)
}

func (x *xmppProcess) describeService() ([]xco.DiscoIdentity, []xco.DiscoFeature) {
	ids := []xco.DiscoIdentity{
		{
			Category: "gateway",
			Type:     "sms",
			Name:     "SMS over XMPP",
		},
	}
	features := []xco.DiscoFeature{
		{
			Var: "urn:xmpp:receipts",
		},
	}
	return ids, features
}

// isFirstContact returns true if the entities local and remote have
// not had any contact during the life of this process.  Contact
// includes both messages and subscriptions.
func (x *xmppProcess) isFirstContact(local, remote *xco.Address) bool {
	if x.contacted == nil {
		return true
	}

	local = local.Bare()
	remotes := x.contacted[*local]
	if remotes == nil {
		return true
	}

	remote = remote.Bare()
	return !remotes[*remote]
}

// hadContact records the fact that local and remote have contacted
// each other.  It could be for the first time or any subsequent time.
func (x *xmppProcess) hadContact(local, remote *xco.Address) {
	local = local.Bare()
	remote = remote.Bare()
	if x.contacted == nil {
		x.contacted = make(map[xco.Address]map[xco.Address]bool)
	}
	remotes := x.contacted[*local]
	if remotes == nil {
		remotes = make(map[xco.Address]bool)
		x.contacted[*local] = remotes
	}
	remotes[*remote] = true
}

func (x *xmppProcess) presenceAvailable(p *xco.Presence) *xco.Presence {
	stanza := &xco.Presence{
		Header: xco.Header{
			From: p.Header.To,
			To:   p.Header.From,
			ID:   NewId(),
		},
	}
	return stanza
}

func (x *xmppProcess) handleSubscribeUnsubscribe(p *xco.Presence) []*xco.Presence {
	// RFC says to use bare JIDs
	p.Header.To.ResourcePart = ""
	p.Header.From.ResourcePart = ""

	stanza := &xco.Presence{
		Header: xco.Header{
			From: p.Header.To,
			To:   p.Header.From,
			ID:   NewId(),
		},
	}
	switch p.Type {
	case "subscribe":
		stanza.Type = "subscribed"
		local := &p.Header.To
		remote := &p.Header.From
		stanzas := []*xco.Presence{
			stanza,
			x.presenceAvailable(p),
		}
		if x.isFirstContact(local, remote) {
			x.hadContact(local, remote)
			//stanzas = append(stanzas, x.requestSubscription(local, remote))
		}
		return stanzas
	case "unsubscribe":
		stanza.Type = "unavailable"
		return []*xco.Presence{stanza}
	}

	return nil
}

func (x *xmppProcess) requestSubscription(local, remote *xco.Address) *xco.Presence {
	stanza := &xco.Presence{
		Header: xco.Header{
			From: *local,
			To:   *remote,
			ID:   NewId(),
		},
		Type: "subscribe",
	}
	return stanza
}
<|endoftext|>


<commit_before>package provisioning

import (
	"context"
	"net/url"
	"os"
	"path/filepath"

	"github.com/n0stack/n0stack/n0core/pkg/driver/qemu_img"
	"github.com/pkg/errors"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	"github.com/golang/protobuf/ptypes/empty"
)

type BlockStorageAgentAPI struct {
	baseDirectory string
}

func CreateBlockStorageAgentAPI(basedir string) (*BlockStorageAgentAPI, error) {
	b, err := filepath.Abs(basedir)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to get absolute path")
	}

	if _, err := os.Stat(b); os.IsNotExist(err) {
		if err := os.MkdirAll(b, 0644); err != nil { // TODO: check permission
			return nil, errors.Wrapf(err, "Failed to mkdir '%s'", b)
		}
	}

	return &BlockStorageAgentAPI{
		baseDirectory: b,
	}, nil
}

func (a *BlockStorageAgentAPI) structPath(name string) string {
	return filepath.Join(a.baseDirectory, name)
}

func (a *BlockStorageAgentAPI) CreateEmptyBlockStorage(ctx context.Context, req *CreateEmptyBlockStorageRequest) (*CreateEmptyBlockStorageResponse, error) {
	path := a.structPath(req.Name)
	res := &CreateEmptyBlockStorageResponse{
		Name:  req.Name,
		Path:  path,
		Bytes: req.Bytes,
	}

	i, err := img.OpenQemuImg(path)
	if err != nil {
		return nil, grpc.Errorf(codes.Internal, "Cannot open '%s': err='%s'", path, err.Error())
	}
	if i.IsExists() {
		return res, grpc.Errorf(codes.AlreadyExists, "")
	}

	if err := i.Create(req.Bytes); err != nil {
		return nil, grpc.Errorf(codes.Internal, "Failed to create image: err='%s'", err.Error())
	}

	return res, nil
}

// タイムアウトが心配
func (a *BlockStorageAgentAPI) FetchBlockStorage(ctx context.Context, req *FetchBlockStorageRequest) (*FetchBlockStorageResponse, error) {
	path := a.structPath(req.Name)
	res := &FetchBlockStorageResponse{
		Name:  req.Name,
		Path:  path,
		Bytes: req.Bytes,
	}

	i, err := img.OpenQemuImg(path)
	if err != nil {
		return nil, grpc.Errorf(codes.Internal, "Cannot open '%s': err='%s'", path, err.Error())
	}
	if i.IsExists() {
		return res, grpc.Errorf(codes.AlreadyExists, "")
	}

	u, err := url.Parse(req.SourceUrl)
	if err != nil {
		return nil, grpc.Errorf(codes.InvalidArgument, "Parsing source_url '%s' is invalid url: err='%s'", req.SourceUrl, err.Error())
	}

	switch u.Scheme {
	case "http", "https":
		if err := i.Download(u); err != nil {
			return nil, grpc.Errorf(codes.Internal, "Failed to download image: err='%s'", err.Error())
		}

	case "file":
		src, err := img.OpenQemuImg(u.Path)
		if err != nil {
			return nil, grpc.Errorf(codes.Internal, "Failed to open source image: err='%s'", err.Error())
		}

		if err := i.Copy(src); err != nil {
			return nil, grpc.Errorf(codes.Internal, "Failed to download image: err='%s'", err.Error())
		}
	}

	return res, nil
}

func (a *BlockStorageAgentAPI) DeleteBlockStorage(ctx context.Context, req *DeleteBlockStorageRequest) (*empty.Empty, error) {
	i, err := img.OpenQemuImg(req.Path)
	if err != nil {
		return nil, grpc.Errorf(codes.Internal, "Cannot open '%s': err='%s'", req.Path, err.Error())
	}
	if !i.IsExists() {
		return nil, grpc.Errorf(codes.NotFound, "")
	}

	if err := i.Delete(); err != nil {
		return nil, grpc.Errorf(codes.Internal, "Failed to delete image: err='%s'", err.Error())
	}

	return &empty.Empty{}, nil
}
<commit_msg>resize qemu image on FetchBlockStorage<commit_after>package provisioning

import (
	"context"
	"net/url"
	"os"
	"path/filepath"

	"github.com/n0stack/n0stack/n0core/pkg/driver/qemu_img"
	"github.com/pkg/errors"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	"github.com/golang/protobuf/ptypes/empty"
)

type BlockStorageAgentAPI struct {
	baseDirectory string
}

func CreateBlockStorageAgentAPI(basedir string) (*BlockStorageAgentAPI, error) {
	b, err := filepath.Abs(basedir)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to get absolute path")
	}

	if _, err := os.Stat(b); os.IsNotExist(err) {
		if err := os.MkdirAll(b, 0644); err != nil { // TODO: check permission
			return nil, errors.Wrapf(err, "Failed to mkdir '%s'", b)
		}
	}

	return &BlockStorageAgentAPI{
		baseDirectory: b,
	}, nil
}

func (a *BlockStorageAgentAPI) structPath(name string) string {
	return filepath.Join(a.baseDirectory, name)
}

func (a *BlockStorageAgentAPI) CreateEmptyBlockStorage(ctx context.Context, req *CreateEmptyBlockStorageRequest) (*CreateEmptyBlockStorageResponse, error) {
	path := a.structPath(req.Name)
	res := &CreateEmptyBlockStorageResponse{
		Name:  req.Name,
		Path:  path,
		Bytes: req.Bytes,
	}

	i, err := img.OpenQemuImg(path)
	if err != nil {
		return nil, grpc.Errorf(codes.Internal, "Cannot open '%s': err='%s'", path, err.Error())
	}
	if i.IsExists() {
		return res, grpc.Errorf(codes.AlreadyExists, "")
	}

	if err := i.Create(req.Bytes); err != nil {
		return nil, grpc.Errorf(codes.Internal, "Failed to create image: err='%s'", err.Error())
	}

	return res, nil
}

// タイムアウトが心配
func (a *BlockStorageAgentAPI) FetchBlockStorage(ctx context.Context, req *FetchBlockStorageRequest) (*FetchBlockStorageResponse, error) {
	path := a.structPath(req.Name)
	res := &FetchBlockStorageResponse{
		Name:  req.Name,
		Path:  path,
		Bytes: req.Bytes,
	}

	i, err := img.OpenQemuImg(path)
	if err != nil {
		return nil, grpc.Errorf(codes.Internal, "Cannot open '%s': err='%s'", path, err.Error())
	}
	if i.IsExists() {
		return res, grpc.Errorf(codes.AlreadyExists, "")
	}

	u, err := url.Parse(req.SourceUrl)
	if err != nil {
		return nil, grpc.Errorf(codes.InvalidArgument, "Parsing source_url '%s' is invalid url: err='%s'", req.SourceUrl, err.Error())
	}

	switch u.Scheme {
	case "http", "https":
		if err := i.Download(u); err != nil {
			return nil, grpc.Errorf(codes.Internal, "Failed to download image: err='%s'", err.Error())
		}

	case "file":
		src, err := img.OpenQemuImg(u.Path)
		if err != nil {
			return nil, grpc.Errorf(codes.Internal, "Failed to open source image: err='%s'", err.Error())
		}

		if err := i.Copy(src); err != nil {
			return nil, grpc.Errorf(codes.Internal, "Failed to download image: err='%s'", err.Error())
		}
	}

	if err := i.Resize(req.Bytes); err != nil {
		return nil, WrapGrpcErrorf(codes.Internal, "Failed to Resize qemu image: err='%s'", err.Error())
	}

	return res, nil
}

func (a *BlockStorageAgentAPI) DeleteBlockStorage(ctx context.Context, req *DeleteBlockStorageRequest) (*empty.Empty, error) {
	i, err := img.OpenQemuImg(req.Path)
	if err != nil {
		return nil, grpc.Errorf(codes.Internal, "Cannot open '%s': err='%s'", req.Path, err.Error())
	}
	if !i.IsExists() {
		return nil, grpc.Errorf(codes.NotFound, "")
	}

	if err := i.Delete(); err != nil {
		return nil, grpc.Errorf(codes.Internal, "Failed to delete image: err='%s'", err.Error())
	}

	return &empty.Empty{}, nil
}
<|endoftext|>


<commit_before>package mesos

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
	"sync"

	"github.com/CiscoCloud/mesos-consul/config"
	"github.com/CiscoCloud/mesos-consul/consul"

	consulapi "github.com/hashicorp/consul/api"
)

type CacheEntry struct {
	service		*consulapi.AgentServiceRegistration
	isRegistered	bool
}

type Mesos struct {
	Consul		*consul.Consul
	Masters		*[]MesosHost
	Lock		sync.Mutex
	ServiceCache	map[string]*CacheEntry
}

func New(c *config.Config, consul *consul.Consul) *Mesos{
	m := new(Mesos)

	if c.Zk == "" {
		return nil
	}

	m.Consul = consul

	m.zkDetector(c.Zk)

	return m
}

func (m *Mesos) Refresh() error {
	sj, err := m.loadState()
	if err != nil {
		log.Print("[ERROR] No master")
		return err
	}

	if (sj.Leader == "") {
		return errors.New("Empty master")
	}

	if m.ServiceCache == nil {
		m.LoadCache()
	}

	m.parseState(sj)

	return nil
}

func (m *Mesos) loadState() (StateJSON, error) {
	var err error
	var sj StateJSON

	defer func() {
		if rec := recover(); rec != nil {
			err = errors.New("can't connect to Mesos")
		}
	}()

	ip, port := m.getLeader()
	if ip == "" {
		return sj, errors.New("No master in zookeeper")
	}

	log.Printf("[INFO] Zookeeper leader: %s:%s", ip, port)

	log.Print("[INFO] reloading from master ", ip)
	sj = m.loadFromMaster(ip, port)

	if rip := leaderIP(sj.Leader); rip != ip {
		log.Print("[WARN] master changed to ", rip)
		sj = m.loadFromMaster(rip, port)
	}

	return sj, err
}

func (m *Mesos) loadFromMaster(ip string, port string) (sj StateJSON) {
	url := "http://" + ip + ":" + port + "/master/state.json"

	req, err := http.NewRequest("GET", url, nil)
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal("[ERROR] ", err)
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[ERROR] ", err)
	}

	err = json.Unmarshal(body, &sj)
	if err != nil {
		log.Fatal("[ERROR] ", err)
	}

	return sj
}

func (m *Mesos) parseState(sj StateJSON) {
	log.Print("[INFO] Running parseState")

	m.RegisterHosts(sj)
	log.Print("[DEBUG] Done running RegisterHosts")

	for _, fw := range sj.Frameworks {
		for _, task := range fw.Tasks {
			host, err := sj.Followers.hostById(task.FollowerId)
			if err == nil && task.State == "TASK_RUNNING" {
				tname := cleanName(task.Name)
				if task.Resources.Ports != "" {
					for _, port := range yankPorts(task.Resources.Ports) {
						m.register(&consulapi.AgentServiceRegistration{
							ID:	fmt.Sprintf("mesos-consul:%s:%s:%d",host,tname,port),
							Name:	tname,
							Port:	port,
							Address: toIP(host),
							})
					}
				}
			}
		}
	}

	// Remove completed tasks
	m.deregister()
}

func yankPorts(ports string) []int {
	rhs := strings.Split(ports, "[")[1]
	lhs := strings.Split(rhs, "]")[0]

	yports := []int{}

	mports := strings.Split(lhs, ",")
	for _,mport := range mports {
		pz := strings.Split(strings.TrimSpace(mport), "-")
		lo, _ := strconv.Atoi(pz[0])
		hi, _ := strconv.Atoi(pz[1])

		for t := lo; t <= hi; t++ {
			yports = append(yports, t)
		}
	}

	return yports
}
<commit_msg>Create the map if it is nil<commit_after>package mesos

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
	"sync"

	"github.com/CiscoCloud/mesos-consul/config"
	"github.com/CiscoCloud/mesos-consul/consul"

	consulapi "github.com/hashicorp/consul/api"
)

type CacheEntry struct {
	service		*consulapi.AgentServiceRegistration
	isRegistered	bool
}

type Mesos struct {
	Consul		*consul.Consul
	Masters		*[]MesosHost
	Lock		sync.Mutex
	ServiceCache	map[string]*CacheEntry
}

func New(c *config.Config, consul *consul.Consul) *Mesos{
	m := new(Mesos)

	if c.Zk == "" {
		return nil
	}

	m.Consul = consul

	m.zkDetector(c.Zk)

	return m
}

func (m *Mesos) Refresh() error {
	sj, err := m.loadState()
	if err != nil {
		log.Print("[ERROR] No master")
		return err
	}

	if (sj.Leader == "") {
		return errors.New("Empty master")
	}

	if m.ServiceCache == nil {
		m.ServiceCache = make(map[string]*CacheEntry)
		m.LoadCache()
	}

	m.parseState(sj)

	return nil
}

func (m *Mesos) loadState() (StateJSON, error) {
	var err error
	var sj StateJSON

	defer func() {
		if rec := recover(); rec != nil {
			err = errors.New("can't connect to Mesos")
		}
	}()

	ip, port := m.getLeader()
	if ip == "" {
		return sj, errors.New("No master in zookeeper")
	}

	log.Printf("[INFO] Zookeeper leader: %s:%s", ip, port)

	log.Print("[INFO] reloading from master ", ip)
	sj = m.loadFromMaster(ip, port)

	if rip := leaderIP(sj.Leader); rip != ip {
		log.Print("[WARN] master changed to ", rip)
		sj = m.loadFromMaster(rip, port)
	}

	return sj, err
}

func (m *Mesos) loadFromMaster(ip string, port string) (sj StateJSON) {
	url := "http://" + ip + ":" + port + "/master/state.json"

	req, err := http.NewRequest("GET", url, nil)
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal("[ERROR] ", err)
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[ERROR] ", err)
	}

	err = json.Unmarshal(body, &sj)
	if err != nil {
		log.Fatal("[ERROR] ", err)
	}

	return sj
}

func (m *Mesos) parseState(sj StateJSON) {
	log.Print("[INFO] Running parseState")

	m.RegisterHosts(sj)
	log.Print("[DEBUG] Done running RegisterHosts")

	for _, fw := range sj.Frameworks {
		for _, task := range fw.Tasks {
			host, err := sj.Followers.hostById(task.FollowerId)
			if err == nil && task.State == "TASK_RUNNING" {
				tname := cleanName(task.Name)
				if task.Resources.Ports != "" {
					for _, port := range yankPorts(task.Resources.Ports) {
						m.register(&consulapi.AgentServiceRegistration{
							ID:	fmt.Sprintf("mesos-consul:%s:%s:%d",host,tname,port),
							Name:	tname,
							Port:	port,
							Address: toIP(host),
							})
					}
				}
			}
		}
	}

	// Remove completed tasks
	m.deregister()
}

func yankPorts(ports string) []int {
	rhs := strings.Split(ports, "[")[1]
	lhs := strings.Split(rhs, "]")[0]

	yports := []int{}

	mports := strings.Split(lhs, ",")
	for _,mport := range mports {
		pz := strings.Split(strings.TrimSpace(mport), "-")
		lo, _ := strconv.Atoi(pz[0])
		hi, _ := strconv.Atoi(pz[1])

		for t := lo; t <= hi; t++ {
			yports = append(yports, t)
		}
	}

	return yports
}
<|endoftext|>


<commit_before>package GoSDK

import (
	"encoding/json"
	"net/url"
)

const (
	_STATS_PREAMBLE  = "/api/v/3/platform/statistics"
	_DBCONN_PREAMBLE = "/api/v/3/platform/dbconnections"
	_LOGS_PREAMBLE   = "/api/v/3/platform/logs"
)

func (u *UserClient) GetConnectedEdgeStatistics(systemKey string, query *Query) (map[string]interface{}, error) {
	creds, err := u.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(u, _STATS_PREAMBLE+"/"+systemKey, q, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) GetConnectedEdgeStatistics(systemKey string, query *Query) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _STATS_PREAMBLE+"/"+systemKey, q, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (u *UserClient) GetConnectedEdgeDBConnections(systemKey string, query *Query) (map[string]interface{}, error) {
	creds, err := u.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(u, _DBCONN_PREAMBLE+"/"+systemKey, q, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) GetConnectedEdgeDBConnections(systemKey string, query *Query) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _DBCONN_PREAMBLE+"/"+systemKey, q, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (u *UserClient) GetConnectedEdgeLogs(systemKey string, query *Query) (map[string]interface{}, error) {
	creds, err := u.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(u, _LOGS_PREAMBLE+"/"+systemKey, q, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func (d *DevClient) GetConnectedEdgeLogs(systemKey string, query *Query) (map[string]interface{}, error) {
	creds, err := d.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(d, _LOGS_PREAMBLE+"/"+systemKey, q, creds, nil)
	resp, err = mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	return resp.Body.(map[string]interface{}), nil
}

func dbQueryToReqQuery(query *Query) (map[string]string, error) {
	var qry map[string]string
	if query != nil {
		query_map := query.serialize()
		query_bytes, err := json.Marshal(query_map)
		if err != nil {
			return nil, err
		}
		qry = map[string]string{
			"query": url.QueryEscape(string(query_bytes)),
		}
	} else {
		qry = nil
	}
	return qry, nil
}
<commit_msg>add platform metrics calls and proxy edge calls through platform clearblade-edge header<commit_after>package GoSDK

import (
	"encoding/json"
	"fmt"
	"net/url"
)

const (
	_StatsPreamble  = "/api/v/3/platform/statistics"
	_DBConnPreamble = "/api/v/3/platform/dbconnections"
	_LogsPreamble   = "/api/v/3/platform/logs"
)

func (u *UserClient) GetPlatformStatistics(systemKey string, query *Query) (map[string]interface{}, error) {
	return getStatistics(u, systemKey, query, nil)
}

func (d *DevClient) GetPlatformStatistics(systemKey string, query *Query) (map[string]interface{}, error) {
	return getStatistics(d, systemKey, query, nil)
}

func (u *UserClient) GetConnectedEdgeStatistics(edgeName, systemKey string, query *Query) (map[string]interface{}, error) {
	return getStatistics(u, systemKey, query, map[string][]string{"ClearBlade-Edge": []string{edgeName}})
}

func (d *DevClient) GetConnectedEdgeStatistics(edgeName, systemKey string, query *Query) (map[string]interface{}, error) {
	return getStatistics(d, systemKey, query, map[string][]string{"ClearBlade-Edge": []string{edgeName}})
}

func getStatistics(c cbClient, systemKey string, query *Query, headers map[string][]string) (map[string]interface{}, error) {
	creds, err := c.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(c, _StatsPreamble+"`"+systemKey, q, creds, headers)
	if err != nil {
		return nil, fmt.Errorf("Unable to communicate with platform, got %s", err)
	}
	mapResp, err := mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	mapBody, ok := mapResp.Body.(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("Expected JSON response, got %#v", resp)
	}
	return mapBody, nil
}

func (u *UserClient) GetPlatformDBConnections(systemKey string, query *Query) (map[string]interface{}, error) {
	return getDBConnections(u, systemKey, query, nil)
}

func (d *DevClient) GetPlatformDBConnections(systemKey string, query *Query) (map[string]interface{}, error) {
	return getDBConnections(d, systemKey, query, nil)
}

func (u *UserClient) GetConnectedEdgeDBConnections(edgeName, systemKey string, query *Query) (map[string]interface{}, error) {
	return getDBConnections(u, systemKey, query, map[string][]string{"ClearBlade-Edge": []string{edgeName}})
}

func (d *DevClient) GetConnectedEdgeDBConnections(edgeName, systemKey string, query *Query) (map[string]interface{}, error) {
	return getDBConnections(d, systemKey, query, map[string][]string{"ClearBlade-Edge": []string{edgeName}})
}

func getDBConnections(c cbClient, systemKey string, query *Query, headers map[string][]string) (map[string]interface{}, error) {
	creds, err := c.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(c, _DBConnPreamble+"/"+systemKey, q, creds, nil)
	if err != nil {
		return nil, fmt.Errorf("Unable to communicate with platform, got %s", err)
	}
	respMap, err := mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	respBody, ok := respMap.Body.(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("Expected JSON response, got %#v", resp)
	}
	return respBody, nil
}

func (u *UserClient) GetPlatformLogs(systemKey string, query *Query) (map[string]interface{}, error) {
	return getLogs(u, systemKey, query, nil)
}

func (d *DevClient) GetPlatformLogs(systemKey string, query *Query) (map[string]interface{}, error) {
	return getLogs(d, systemKey, query, nil)
}

func (u *UserClient) GetConnectedEdgeLogs(edgeName, systemKey string, query *Query) (map[string]interface{}, error) {
	return getLogs(u, systemKey, query, map[string][]string{"ClearBlade-Edge": []string{edgeName}})
}

func (d *DevClient) GetConnectedEdgeLogs(edgeName, systemKey string, query *Query) (map[string]interface{}, error) {
	return getLogs(d, systemKey, query, map[string][]string{"ClearBlade-Edge": []string{edgeName}})
}

func getLogs(c cbClient, systemKey string, query *Query, headers map[string][]string) (map[string]interface{}, error) {
	creds, err := c.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(c, _LogsPreamble+"/"+systemKey, q, creds, nil)
	if err != nil {
		return nil, fmt.Errorf("Unable to communicate with platform, got %s", err)
	}
	respMap, err := mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	respBody, ok := respMap.Body.(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("Expected JSON response, got %#v", resp)
	}
	return respBody, nil
}

func dbQueryToReqQuery(query *Query) (map[string]string, error) {
	var qry map[string]string
	if query != nil {
		queryMap := query.serialize()
		queryBytes, err := json.Marshal(queryMap)
		if err != nil {
			return nil, err
		}
		qry = map[string]string{
			"query": url.QueryEscape(string(queryBytes)),
		}
	} else {
		qry = nil
	}
	return qry, nil
}
<|endoftext|>


<commit_before>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package consensus

import (
	"github.com/openblockchain/obc-peer/openchain/ledger/statemgmt"
	pb "github.com/openblockchain/obc-peer/protos"
)

// Consenter is used to receive messages from the network
// Every consensus plugin needs to implement this interface
type Consenter interface {
	RecvMsg(msg *pb.OpenchainMessage) error
}

// Communicator is used to send messages to other validators
type Communicator interface {
	GetNetworkInfo() (self *pb.PeerEndpoint, network []*pb.PeerEndpoint, err error)
	GetNetworkHandles() (self *pb.PeerID, network []*pb.PeerID, err error)
	Broadcast(msg *pb.OpenchainMessage) error
	Unicast(msg *pb.OpenchainMessage, receiverHandle *pb.PeerID) error
}

// SecurityUtils is used to access the sign/verify methods from the crypto package
type SecurityUtils interface {
	Sign(msg []byte) ([]byte, error)
	Verify(peerID *pb.PeerID, signature []byte, message []byte) error
}

// ReadOnlyLedger is used for interrogating the blockchain
type ReadOnlyLedger interface {
	GetBlock(id uint64) (block *pb.Block, err error)
	GetCurrentStateHash() (stateHash []byte, err error)
	GetBlockchainSize() (uint64, error)
}

// UtilLedger contains additional useful utility functions for interrogating the blockchain
type UtilLedger interface {
	HashBlock(block *pb.Block) ([]byte, error)
	VerifyBlockchain(start, finish uint64) (uint64, error)
}

// WritableLedger is useful for updating the blockchain during state transfer
type WritableLedger interface {
	PutBlock(blockNumber uint64, block *pb.Block) error
	ApplyStateDelta(id interface{}, delta *statemgmt.StateDelta) error
	CommitStateDelta(id interface{}) error
	RollbackStateDelta(id interface{}) error
	EmptyState() error
}

// Ledger is an unrestricted union of reads, utilities, and updates
type Ledger interface {
	ReadOnlyLedger
	UtilLedger
	WritableLedger
}

// Executor is used to invoke transactions, potentially modifying the backing ledger
type Executor interface {
	BeginTxBatch(id interface{}) error
	ExecTXs(txs []*pb.Transaction) ([]byte, []error)
	CommitTxBatch(id interface{}, transactions []*pb.Transaction, transactionsResults []*pb.TransactionResult, metadata []byte) error
	RollbackTxBatch(id interface{}) error
	PreviewCommitTxBatchBlock(id interface{}, transactions []*pb.Transaction, metadata []byte) (*pb.Block, error)
}

// RemoteLedgers is used to interrogate the blockchain of other replicas
type RemoteLedgers interface {
	GetRemoteBlocks(replicaID *pb.PeerID, start, finish uint64) (<-chan *pb.SyncBlocks, error)
	GetRemoteStateSnapshot(replicaID *pb.PeerID) (<-chan *pb.SyncStateSnapshot, error)
	GetRemoteStateDeltas(replicaID *pb.PeerID, start, finish uint64) (<-chan *pb.SyncStateDeltas, error)
}

// LedgerStack serves as interface to the blockchain-oriented activities, such as executing transactions, querying, and updating the ledger
type LedgerStack interface {
	Executor
	Ledger
	RemoteLedgers
}

// CPI (Consensus Programming Interface) is the set of stack-facing methods available to the consensus plugin
type CPI interface {
	Communicator
	SecurityUtils
	LedgerStack
}
<commit_msg>Add Inquirer interface to CPI<commit_after>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

package consensus

import (
	"github.com/openblockchain/obc-peer/openchain/ledger/statemgmt"
	pb "github.com/openblockchain/obc-peer/protos"
)

// Consenter is used to receive messages from the network
// Every consensus plugin needs to implement this interface
type Consenter interface {
	RecvMsg(msg *pb.OpenchainMessage) error
}

// Inquirer is used to retrieve info about the validating network
type Inquirer interface {
	GetNetworkInfo() (self *pb.PeerEndpoint, network []*pb.PeerEndpoint, err error)
	GetNetworkHandles() (self *pb.PeerID, network []*pb.PeerID, err error)
}

// Communicator is used to send messages to other validators
type Communicator interface {
	Broadcast(msg *pb.OpenchainMessage) error
	Unicast(msg *pb.OpenchainMessage, receiverHandle *pb.PeerID) error
}

// SecurityUtils is used to access the sign/verify methods from the crypto package
type SecurityUtils interface {
	Sign(msg []byte) ([]byte, error)
	Verify(peerID *pb.PeerID, signature []byte, message []byte) error
}

// ReadOnlyLedger is used for interrogating the blockchain
type ReadOnlyLedger interface {
	GetBlock(id uint64) (block *pb.Block, err error)
	GetCurrentStateHash() (stateHash []byte, err error)
	GetBlockchainSize() (uint64, error)
}

// UtilLedger contains additional useful utility functions for interrogating the blockchain
type UtilLedger interface {
	HashBlock(block *pb.Block) ([]byte, error)
	VerifyBlockchain(start, finish uint64) (uint64, error)
}

// WritableLedger is useful for updating the blockchain during state transfer
type WritableLedger interface {
	PutBlock(blockNumber uint64, block *pb.Block) error
	ApplyStateDelta(id interface{}, delta *statemgmt.StateDelta) error
	CommitStateDelta(id interface{}) error
	RollbackStateDelta(id interface{}) error
	EmptyState() error
}

// Ledger is an unrestricted union of reads, utilities, and updates
type Ledger interface {
	ReadOnlyLedger
	UtilLedger
	WritableLedger
}

// Executor is used to invoke transactions, potentially modifying the backing ledger
type Executor interface {
	BeginTxBatch(id interface{}) error
	ExecTXs(txs []*pb.Transaction) ([]byte, []error)
	CommitTxBatch(id interface{}, transactions []*pb.Transaction, transactionsResults []*pb.TransactionResult, metadata []byte) error
	RollbackTxBatch(id interface{}) error
	PreviewCommitTxBatchBlock(id interface{}, transactions []*pb.Transaction, metadata []byte) (*pb.Block, error)
}

// RemoteLedgers is used to interrogate the blockchain of other replicas
type RemoteLedgers interface {
	GetRemoteBlocks(replicaID *pb.PeerID, start, finish uint64) (<-chan *pb.SyncBlocks, error)
	GetRemoteStateSnapshot(replicaID *pb.PeerID) (<-chan *pb.SyncStateSnapshot, error)
	GetRemoteStateDeltas(replicaID *pb.PeerID, start, finish uint64) (<-chan *pb.SyncStateDeltas, error)
}

// LedgerStack serves as interface to the blockchain-oriented activities, such as executing transactions, querying, and updating the ledger
type LedgerStack interface {
	Executor
	Ledger
	RemoteLedgers
}

// CPI (Consensus Programming Interface) is the set of stack-facing methods available to the consensus plugin
type CPI interface {
	Inquirer
	Communicator
	SecurityUtils
	LedgerStack
}
<|endoftext|>


<commit_before>package minion

import (
	"os"
	"os/signal"
	"log"
	"time"
	"strings"
	"strconv"
	"path/filepath"
	"encoding/json"

	"code.google.com/p/go-uuid/uuid"
	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	"github.com/coreos/etcd/client"
)

// Root keyspace in etcd
const etcdKeySpace = "/gru"

// Etcd Minion
type EtcdMinion struct {
	// Name of this minion
	Name string

	// Minion root node in etcd
	MinionRootDir string

	// Minion queue node in etcd
	QueueDir string

	// Root node for results in etcd
	//ResultRootDir string

	// Root node for classifiers in etcd
	ClassifierDir string

	// Minion unique identifier
	UUID uuid.UUID

	// KeysAPI client to etcd
	KAPI client.KeysAPI
}

// Etcd minion task
type EtcdTask struct {
	// Command to be executed by the minion
	Command string

	// Time when the command was sent for processing
	Timestamp int64

	// Task unique identifier
	UUID uuid.UUID
}

func NewEtcdTask(command string, timestamp int64) MinionTask {
	t := &EtcdTask{
		Command: command,
		Timestamp: timestamp,
		UUID: uuid.NewRandom(),
	}

	return t
}

// Gets the task unique identifier
func (t *EtcdTask) GetUUID() uuid.UUID {
	return t.UUID
}

// Gets the task command to be executed
func (t *EtcdTask) GetCommand() (string, error) {
	return t.Command, nil
}

// Returns the time a task has been received for processing
func (t *EtcdTask) GetTimestamp() (int64, error) {
	return t.Timestamp, nil
}

// Processes a task
func (t *EtcdTask) Process() error {
	id := t.GetUUID()
	log.Printf("Processing task %s\n", id.String())

	// TODO: Actually process the command

	return nil
}

// Create a new minion
func NewEtcdMinion(name string, kapi client.KeysAPI) Minion {
	minionUUID := uuid.NewSHA1(uuid.NameSpace_DNS, []byte(name))
	minionRootDir := filepath.Join(etcdKeySpace, "minion", minionUUID.String())
	queueDir := filepath.Join(minionRootDir, "queue")
//	resultDir := filepath.Join(...)
	classifierDir := filepath.Join(minionRootDir, "classifier")

	log.Printf("Created minion with uuid %s\n", minionUUID)

	m := &EtcdMinion{
		Name: name,
		MinionRootDir: minionRootDir,
		QueueDir: queueDir,
//		ResultRootDir: resultRootDir,
		ClassifierDir: classifierDir,
		UUID: minionUUID,
		KAPI: kapi,
	}

	return m
}

// Get the minion UUID
func (m *EtcdMinion) GetUUID() uuid.UUID {
	return m.UUID
}

// Get the human-readable name of the minion
func (m *EtcdMinion) GetName() (string, error) {
	return m.Name, nil
}

// Set the human-readable name of the minion
func (m *EtcdMinion) SetName(name string) error {
	key := filepath.Join(m.MinionRootDir, "name")
	opts := &client.SetOptions{
		PrevExist: client.PrevIgnore,
	}

	_, err := m.KAPI.Set(context.Background(), key, m.Name, opts)

	return err
}

// Set the time the minion was last seen in seconds since the Epoch
func (m *EtcdMinion) SetLastseen(s int64) error {
	key := filepath.Join(m.MinionRootDir, "lastseen")
	lastseen := strconv.FormatInt(s, 10)
	opts := &client.SetOptions{
		PrevExist: client.PrevIgnore,
	}

	_, err := m.KAPI.Set(context.Background(), key, lastseen, opts)

	return err
}

// Get classifier for a minion
func (m *EtcdMinion) GetClassifier(key string) (MinionClassifier, error) {
	klassifier := new(SimpleClassifier)
	klassifierNode := filepath.Join(m.ClassifierDir, key, "info")

	// Get classifier from etcd and deserialize
	resp, err := m.KAPI.Get(context.Background(), klassifierNode, nil)

	if err != nil {
		return klassifier, err
	}

	err = json.Unmarshal([]byte(resp.Node.Value), &klassifier)

	return klassifier, err
}

// Classify a minion  a given key and value
func (m *EtcdMinion) SetClassifier(c MinionClassifier) error {
	// Classifiers in etcd expire after an hour
	opts := &client.SetOptions{
		PrevExist: client.PrevIgnore,
		TTL: time.Hour,
	}

	// Get classifier values
	key, err := c.GetKey()
	description, err := c.GetDescription()
	value, err := c.GetValue(m)

	if err != nil {
		return err
	}

	// Create a simple classifier and serialize to JSON
	klassifier := NewSimpleClassifier(key, value, description)
	data, err := json.Marshal(klassifier)

	if err != nil {
		log.Printf("Failed to serialize classifier: %s\n", key)
		return err
	}

	// Set classifier in etcd
	klassifierNode := filepath.Join(m.ClassifierDir, key, "info")
	_, err = m.KAPI.Set(context.Background(), klassifierNode, string(data), opts)

	return err
}

// Runs periodic jobs such as refreshing classifiers and updating lastseen
func (m *EtcdMinion) Refresh(ticker *time.Ticker) error {
	for {
		// Update classifiers
		for _, classifier := range ClassifierRegistry {
			m.SetClassifier(classifier)
		}

		// Update lastseen time
		now := time.Now().Unix()
		err := m.SetLastseen(now)
		if err != nil {
			log.Printf("Failed to update lastseen time: %s\n", err)
		}

		<- ticker.C
	}

	return nil
}

// Unmarshals task from etcd and removes it from the queue
func UnmarshalEtcdTask(node *client.Node) (*EtcdTask, error) {
	task := new(EtcdTask)
	err := json.Unmarshal([]byte(node.Value), &task)

	if err != nil {
		log.Printf("Invalid task: key: %s\n", node.Key)
		log.Printf("Invalid task: value: %s\n", node.Value)
		log.Printf("Invalid task: error: %s\n", err)
	}

	return task, err
}

// Monitors etcd for new tasks for processing
func (m *EtcdMinion) TaskListener(c chan<- MinionTask) error {
	log.Printf("Starting task listener")

	watcherOpts := &client.WatcherOptions{
		Recursive: true,
	}
	watcher := m.KAPI.Watcher(m.QueueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			log.Printf("Failed to read task: %s\n", err)
			continue
		}

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		task, err := UnmarshalEtcdTask(resp.Node)
		m.KAPI.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			continue
		}

		c <- task
	}

	return nil
}

// Processes new tasks
func (m *EtcdMinion) TaskRunner(c <-chan MinionTask) error {
	for {
		task := <-c
		task.Process()
	}

	return nil
}

// Checks for any tasks in backlog
func (m *EtcdMinion) CheckForBacklog(c chan<- MinionTask) error {
	opts := &client.GetOptions{
		Recursive: true,
		Sort: true,
	}

	// Get backlog tasks if any
	resp, err := m.KAPI.Get(context.Background(), m.QueueDir, opts)
	if err != nil {
		return nil
	}

	backlog := resp.Node.Nodes

	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d tasks in backlog", len(backlog))
	for _, node := range backlog {
		task, err := UnmarshalEtcdTask(node)
		m.KAPI.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		c <- task
	}

	return nil
}

// Main entry point of the minion
func (m *EtcdMinion) Serve() error {
	// Channel on which we send the quit signal
	quit := make(chan os.Signal, 1)
	signal.Notify(quit, os.Interrupt)

	// Run any periodic tasks every hour
	ticker := time.NewTicker(time.Minute * 15)
	go m.Refresh(ticker)

	// Check for backlog tasks and start task listener and runner
	tasks := make(chan MinionTask)
	go m.TaskListener(tasks)
	go m.CheckForBacklog(tasks)
	go m.TaskRunner(tasks)

	// Block until a stop signal is received
	s := <-quit
	log.Printf("Received %s signal, shutting down", s)
	close(quit)
	close(tasks)

	return nil
}

<commit_msg>EtcdKeySpace const is exported so that it can be used by clients as well<commit_after>package minion

import (
	"os"
	"os/signal"
	"log"
	"time"
	"strings"
	"strconv"
	"path/filepath"
	"encoding/json"

	"code.google.com/p/go-uuid/uuid"
	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	"github.com/coreos/etcd/client"
)

// Root keyspace in etcd
const EtcdKeySpace = "/gru"

// Etcd Minion
type EtcdMinion struct {
	// Name of this minion
	Name string

	// Minion root node in etcd
	MinionRootDir string

	// Minion queue node in etcd
	QueueDir string

	// Root node for results in etcd
	//ResultRootDir string

	// Root node for classifiers in etcd
	ClassifierDir string

	// Minion unique identifier
	UUID uuid.UUID

	// KeysAPI client to etcd
	KAPI client.KeysAPI
}

// Etcd minion task
type EtcdTask struct {
	// Command to be executed by the minion
	Command string

	// Time when the command was sent for processing
	Timestamp int64

	// Task unique identifier
	UUID uuid.UUID
}

func NewEtcdTask(command string, timestamp int64) MinionTask {
	t := &EtcdTask{
		Command: command,
		Timestamp: timestamp,
		UUID: uuid.NewRandom(),
	}

	return t
}

// Gets the task unique identifier
func (t *EtcdTask) GetUUID() uuid.UUID {
	return t.UUID
}

// Gets the task command to be executed
func (t *EtcdTask) GetCommand() (string, error) {
	return t.Command, nil
}

// Returns the time a task has been received for processing
func (t *EtcdTask) GetTimestamp() (int64, error) {
	return t.Timestamp, nil
}

// Processes a task
func (t *EtcdTask) Process() error {
	id := t.GetUUID()
	log.Printf("Processing task %s\n", id.String())

	// TODO: Actually process the command

	return nil
}

// Create a new minion
func NewEtcdMinion(name string, kapi client.KeysAPI) Minion {
	minionUUID := uuid.NewSHA1(uuid.NameSpace_DNS, []byte(name))
	minionRootDir := filepath.Join(EtcdKeySpace, "minion", minionUUID.String())
	queueDir := filepath.Join(minionRootDir, "queue")
//	resultDir := filepath.Join(...)
	classifierDir := filepath.Join(minionRootDir, "classifier")

	log.Printf("Created minion with uuid %s\n", minionUUID)

	m := &EtcdMinion{
		Name: name,
		MinionRootDir: minionRootDir,
		QueueDir: queueDir,
//		ResultRootDir: resultRootDir,
		ClassifierDir: classifierDir,
		UUID: minionUUID,
		KAPI: kapi,
	}

	return m
}

// Get the minion UUID
func (m *EtcdMinion) GetUUID() uuid.UUID {
	return m.UUID
}

// Get the human-readable name of the minion
func (m *EtcdMinion) GetName() (string, error) {
	return m.Name, nil
}

// Set the human-readable name of the minion
func (m *EtcdMinion) SetName(name string) error {
	key := filepath.Join(m.MinionRootDir, "name")
	opts := &client.SetOptions{
		PrevExist: client.PrevIgnore,
	}

	_, err := m.KAPI.Set(context.Background(), key, m.Name, opts)

	return err
}

// Set the time the minion was last seen in seconds since the Epoch
func (m *EtcdMinion) SetLastseen(s int64) error {
	key := filepath.Join(m.MinionRootDir, "lastseen")
	lastseen := strconv.FormatInt(s, 10)
	opts := &client.SetOptions{
		PrevExist: client.PrevIgnore,
	}

	_, err := m.KAPI.Set(context.Background(), key, lastseen, opts)

	return err
}

// Get classifier for a minion
func (m *EtcdMinion) GetClassifier(key string) (MinionClassifier, error) {
	klassifier := new(SimpleClassifier)
	klassifierNode := filepath.Join(m.ClassifierDir, key, "info")

	// Get classifier from etcd and deserialize
	resp, err := m.KAPI.Get(context.Background(), klassifierNode, nil)

	if err != nil {
		return klassifier, err
	}

	err = json.Unmarshal([]byte(resp.Node.Value), &klassifier)

	return klassifier, err
}

// Classify a minion  a given key and value
func (m *EtcdMinion) SetClassifier(c MinionClassifier) error {
	// Classifiers in etcd expire after an hour
	opts := &client.SetOptions{
		PrevExist: client.PrevIgnore,
		TTL: time.Hour,
	}

	// Get classifier values
	key, err := c.GetKey()
	description, err := c.GetDescription()
	value, err := c.GetValue(m)

	if err != nil {
		return err
	}

	// Create a simple classifier and serialize to JSON
	klassifier := NewSimpleClassifier(key, value, description)
	data, err := json.Marshal(klassifier)

	if err != nil {
		log.Printf("Failed to serialize classifier: %s\n", key)
		return err
	}

	// Set classifier in etcd
	klassifierNode := filepath.Join(m.ClassifierDir, key, "info")
	_, err = m.KAPI.Set(context.Background(), klassifierNode, string(data), opts)

	return err
}

// Runs periodic jobs such as refreshing classifiers and updating lastseen
func (m *EtcdMinion) Refresh(ticker *time.Ticker) error {
	for {
		// Update classifiers
		for _, classifier := range ClassifierRegistry {
			m.SetClassifier(classifier)
		}

		// Update lastseen time
		now := time.Now().Unix()
		err := m.SetLastseen(now)
		if err != nil {
			log.Printf("Failed to update lastseen time: %s\n", err)
		}

		<- ticker.C
	}

	return nil
}

// Unmarshals task from etcd and removes it from the queue
func UnmarshalEtcdTask(node *client.Node) (*EtcdTask, error) {
	task := new(EtcdTask)
	err := json.Unmarshal([]byte(node.Value), &task)

	if err != nil {
		log.Printf("Invalid task: key: %s\n", node.Key)
		log.Printf("Invalid task: value: %s\n", node.Value)
		log.Printf("Invalid task: error: %s\n", err)
	}

	return task, err
}

// Monitors etcd for new tasks for processing
func (m *EtcdMinion) TaskListener(c chan<- MinionTask) error {
	log.Printf("Starting task listener")

	watcherOpts := &client.WatcherOptions{
		Recursive: true,
	}
	watcher := m.KAPI.Watcher(m.QueueDir, watcherOpts)

	for {
		resp, err := watcher.Next(context.Background())
		if err != nil {
			log.Printf("Failed to read task: %s\n", err)
			continue
		}

		// Ignore "delete" events when removing a task from the queue
		action := strings.ToLower(resp.Action)
		if strings.EqualFold(action, "delete") {
			continue
		}

		task, err := UnmarshalEtcdTask(resp.Node)
		m.KAPI.Delete(context.Background(), resp.Node.Key, nil)

		if err != nil {
			continue
		}

		c <- task
	}

	return nil
}

// Processes new tasks
func (m *EtcdMinion) TaskRunner(c <-chan MinionTask) error {
	for {
		task := <-c
		task.Process()
	}

	return nil
}

// Checks for any tasks in backlog
func (m *EtcdMinion) CheckForBacklog(c chan<- MinionTask) error {
	opts := &client.GetOptions{
		Recursive: true,
		Sort: true,
	}

	// Get backlog tasks if any
	resp, err := m.KAPI.Get(context.Background(), m.QueueDir, opts)
	if err != nil {
		return nil
	}

	backlog := resp.Node.Nodes

	if len(backlog) == 0 {
		// No backlog tasks found
		return nil
	}

	log.Printf("Found %d tasks in backlog", len(backlog))
	for _, node := range backlog {
		task, err := UnmarshalEtcdTask(node)
		m.KAPI.Delete(context.Background(), node.Key, nil)

		if err != nil {
			continue
		}

		c <- task
	}

	return nil
}

// Main entry point of the minion
func (m *EtcdMinion) Serve() error {
	// Channel on which we send the quit signal
	quit := make(chan os.Signal, 1)
	signal.Notify(quit, os.Interrupt)

	// Run any periodic tasks every hour
	ticker := time.NewTicker(time.Minute * 15)
	go m.Refresh(ticker)

	// Check for backlog tasks and start task listener and runner
	tasks := make(chan MinionTask)
	go m.TaskListener(tasks)
	go m.CheckForBacklog(tasks)
	go m.TaskRunner(tasks)

	// Block until a stop signal is received
	s := <-quit
	log.Printf("Received %s signal, shutting down", s)
	close(quit)
	close(tasks)

	return nil
}

<|endoftext|>