<commit_before>package logberry

import (
	"sync/atomic"
)

// Task represents a particular component, function, or activity.  In
// general a Task is meant to be used within a single thread of
// execution, and the calling code is responsible for managing any
// concurrent manipulation.
type Task struct {
	uid uint64

	root *Root

	parent *Task

	component string

	activity string

	data D
}

var numtasks uint64

func newtaskuid() uint64 {
	// We have seen this atomic call cause problems on ARM...
	return atomic.AddUint64(&numtasks, 1) - 1
}

func newtask(parent *Task, component string, activity string, data []interface{}) *Task {

	t := &Task{
		uid:      newtaskuid(),
		parent:   parent,
		activity: activity,
	}

	if parent != nil {
		t.root = parent.root
		t.component = parent.component
	} else {
		t.root = Std
	}

	if component != "" {
		t.component = component
	}

	t.root.event(t, BEGIN, t.activity+" begin", DAggregate(data))

	return t

}

// Task creates a new sub-task.  Parameter activity should be a short
// natural language description of the work that the Task represents,
// without any terminating punctuation.
func (x *Task) Task(activity string, data ...interface{}) *Task {
	return newtask(x, "", activity, data)
}

// Component creates a new Task object representing related long-lived
// functionality, rather than a directed, tightly scoped line of
// computation.  Parameter component should be a short lowercase
// string identifying the class, module, or other component that this
// Task represents.  The activity text of this Task is set to be
// "Component " + component.
func (x *Task) Component(component string, data ...interface{}) *Task {
	return newtask(x, component, "Component "+component, data)
}

// AddData incorporates the given data into that associated and
// reported with this Task.  The rules for this construction are
// explained in CopyFrom.  This call does not generate a log event.
// The host Task is passed through as the return.  Among other things,
// this function is useful to silently accumulate data into the Task
// as it proceeds, to be reported when it concludes.
func (x *Task) AddData(data ...interface{}) *Task {
	x.data.CopyFrom(data)
	return x
}

// Event generates a user-specified log event.  Parameter event tags
// the class of the event, generally a short lowercase whitespace-free
// identifier.  A human-oriented text message is given as the msg
// parameter.  This should generally be static, short, use sentence
// capitalization but no terminating punctuation, and not itself
// include any data, which is better left to the structured data.  The
// variadic data parameter is aggregated as a D and reported with the
// event, as is the data permanently associated with the Task.  The
// given data is not associated to the Task permanently.
func (x *Task) Event(event string, msg string, data ...interface{}) {
	x.root.event(x, event, msg, DAggregate(data).CopyFrom(x.data))
}

// Info generates an informational log event.  A human-oriented text
// message is given as the msg parameter.  This should generally be
// static, short, use sentence capitalization but no terminating
// punctuation, and not itself include any data, which is better left
// to the structured data.  The variadic data parameter is aggregated
// as a D and reported with the event, as is the data permanently
// associated with the Task.  The given data is not associated to the
// Task permanently.
func (x *Task) Info(msg string, data ...interface{}) {
	x.root.event(x, INFO, msg, DAggregate(data).CopyFrom(x.data))
}

// Warning generates a warning log event indicating that a fault was
// encountered but the task is proceeding acceptably.  This should
// generally be static, short, use sentence capitalization but no
// terminating punctuation, and not itself include any data, which is
// better left to the structured data.  The variadic data parameter is
// aggregated as a D and reported with the event, as is the data
// permanently associated with the Task.  The given data is not
// associated to the Task permanently.
func (x *Task) Warning(msg string, data ...interface{}) {
	x.root.event(x, WARNING, msg, DAggregate(data).CopyFrom(x.data))
}

// Ready generates a ready log event reporting that the activity or
// component the Task represents is initialized and prepared to begin.
// The variadic data parameter is aggregated as a D and reported with
// the event, as is the data permanently associated with the Task.
// The given data is not associated to the Task permanently.
func (x *Task) Ready(data ...interface{}) {
	x.root.event(x, READY, x.activity+" ready",
		DAggregate(data).CopyFrom(x.data))
}

// Stopped generates a stopped log event reporting that the activity
// or component the Task represents has paused or shutdown.  The
// variadic data parameter is aggregated as a D and reported with the
// event, as is the data permanently associated with the Task.  The
// given data is not associated to the Task permanently.
func (x *Task) Stopped(data ...interface{}) {
	x.root.event(x, STOPPED, x.activity+" stopped",
		DAggregate(data).CopyFrom(x.data))
}

// Finalized generates an end log event reporting that the component
// the Task represents has ceased.  It is generally intended to be
// used for components, while Success is used for discrete activities.
// Continuing to use the Task is discouraged.  The variadic data
// parameter is aggregated as a D and reported with the event, as is
// the data permanently associated with the Task.  The given data is
// not associated to the Task permanently.
func (x *Task) Finalized(data ...interface{}) {
	x.root.event(x, END, x.activity+" finalized", DAggregate(data).CopyFrom(x.data))
}

// Success generates a success log event reporting that the activity
// the Task represents has concluded successfully.  It always returns
// nil.  Continuing to use the Task is discouraged.  The variadic data
// parameter is aggregated as a D and reported with the event, as is
// the data permanently associated with the Task.  The given data is
// not associated to the Task permanently.
func (x *Task) Success(data ...interface{}) error {
	x.root.event(x, SUCCESS, x.activity+" success", DAggregate(data).CopyFrom(x.data))
	return nil
}

// Error generates an error log event reporting an unrecoverable fault
// in an activity or component.  An error is returned wrapping the
// original error with a message reporting that the Task's activity
// has failed.  Continuing to use the Task is discouraged.  The
// variadic data parameter is aggregated as a D and embedded in the
// generated error.  It and the data permanently associated with the
// Task is reported with the event.  The reported source code position
// of the generated task error is adjusted to be the event invocation.
func (x *Task) Error(err error, data ...interface{}) error {

	m := x.activity + " failed"

	e := wraperror(m, err, data)
	e.Locate(1)
	e.reported = true

	rep := D{
		"Source": e.Source,
	}

	if ex, ok := err.(*Error); !ok || !ex.reported {
		rep["Cause"] = DAggregate([]interface{}{err})
	}

	rep.CopyFrom(DAggregate(data))
	rep.CopyFrom(x.data)

	x.root.event(x, ERROR, m, rep)

	return e
}

func (x *Task) WrapError(msg string, source error, data ...interface{}) error {
	err := wraperror(msg, source, nil)
	err.Locate(1)

	m := x.activity + " failed"
	x.root.event(x, ERROR, m, D{"Error": DAggregate([]interface{}{err})}.CopyFrom(DAggregate(data)).CopyFrom(x.data))

	e := wraperror(m, err, data)
	e.Locate(1)
	return e
}

// Failure generates an error log event reporting an unrecoverable
// fault.  Failure and Error are essentially the same, the difference
// being that Failure is the first point of fault while Error takes an
// underlying error typically returned from another function or
// component.  An error is returned reporting that the activity or
// component represented by the Task has failed due to the underlying
// cause given in the message.  Continuing to use the Task is
// discouraged.  The variadic data parameter is aggregated as a D and
// embedded in the generated task error.  It and the data permanently
// associated with the Task is reported with the event.  The reported
// source code position of the generated task error is adjusted to be
// the event invocation.
func (x *Task) Failure(msg string, data ...interface{}) error {
	err := newerror(msg, nil)
	err.Locate(1)

	m := x.activity + " failed"
	x.root.event(x, ERROR, m, D{"Error": DAggregate([]interface{}{err})}.CopyFrom(DAggregate(data)).CopyFrom(x.data))

	e := wraperror(m, err, data)
	e.Locate(1)
	return e
}
<commit_msg>Semantics change to Task()/Component() data meant data was no longer initialized.<commit_after>package logberry

import (
	"sync/atomic"
)

// Task represents a particular component, function, or activity.  In
// general a Task is meant to be used within a single thread of
// execution, and the calling code is responsible for managing any
// concurrent manipulation.
type Task struct {
	uid uint64

	root *Root

	parent *Task

	component string

	activity string

	data D
}

var numtasks uint64

func newtaskuid() uint64 {
	// We have seen this atomic call cause problems on ARM...
	return atomic.AddUint64(&numtasks, 1) - 1
}

func newtask(parent *Task, component string, activity string, data []interface{}) *Task {

	t := &Task{
		uid:      newtaskuid(),
		parent:   parent,
		activity: activity,
		data: D{},
	}

	if parent != nil {
		t.root = parent.root
		t.component = parent.component
	} else {
		t.root = Std
	}

	if component != "" {
		t.component = component
	}

	t.root.event(t, BEGIN, t.activity+" begin", DAggregate(data))

	return t

}

// Task creates a new sub-task.  Parameter activity should be a short
// natural language description of the work that the Task represents,
// without any terminating punctuation.
func (x *Task) Task(activity string, data ...interface{}) *Task {
	return newtask(x, "", activity, data)
}

// Component creates a new Task object representing related long-lived
// functionality, rather than a directed, tightly scoped line of
// computation.  Parameter component should be a short lowercase
// string identifying the class, module, or other component that this
// Task represents.  The activity text of this Task is set to be
// "Component " + component.
func (x *Task) Component(component string, data ...interface{}) *Task {
	return newtask(x, component, "Component "+component, data)
}

// AddData incorporates the given data into that associated and
// reported with this Task.  The rules for this construction are
// explained in CopyFrom.  This call does not generate a log event.
// The host Task is passed through as the return.  Among other things,
// this function is useful to silently accumulate data into the Task
// as it proceeds, to be reported when it concludes.
func (x *Task) AddData(data ...interface{}) *Task {
	x.data.CopyFrom(data)
	return x
}

// Event generates a user-specified log event.  Parameter event tags
// the class of the event, generally a short lowercase whitespace-free
// identifier.  A human-oriented text message is given as the msg
// parameter.  This should generally be static, short, use sentence
// capitalization but no terminating punctuation, and not itself
// include any data, which is better left to the structured data.  The
// variadic data parameter is aggregated as a D and reported with the
// event, as is the data permanently associated with the Task.  The
// given data is not associated to the Task permanently.
func (x *Task) Event(event string, msg string, data ...interface{}) {
	x.root.event(x, event, msg, DAggregate(data).CopyFrom(x.data))
}

// Info generates an informational log event.  A human-oriented text
// message is given as the msg parameter.  This should generally be
// static, short, use sentence capitalization but no terminating
// punctuation, and not itself include any data, which is better left
// to the structured data.  The variadic data parameter is aggregated
// as a D and reported with the event, as is the data permanently
// associated with the Task.  The given data is not associated to the
// Task permanently.
func (x *Task) Info(msg string, data ...interface{}) {
	x.root.event(x, INFO, msg, DAggregate(data).CopyFrom(x.data))
}

// Warning generates a warning log event indicating that a fault was
// encountered but the task is proceeding acceptably.  This should
// generally be static, short, use sentence capitalization but no
// terminating punctuation, and not itself include any data, which is
// better left to the structured data.  The variadic data parameter is
// aggregated as a D and reported with the event, as is the data
// permanently associated with the Task.  The given data is not
// associated to the Task permanently.
func (x *Task) Warning(msg string, data ...interface{}) {
	x.root.event(x, WARNING, msg, DAggregate(data).CopyFrom(x.data))
}

// Ready generates a ready log event reporting that the activity or
// component the Task represents is initialized and prepared to begin.
// The variadic data parameter is aggregated as a D and reported with
// the event, as is the data permanently associated with the Task.
// The given data is not associated to the Task permanently.
func (x *Task) Ready(data ...interface{}) {
	x.root.event(x, READY, x.activity+" ready",
		DAggregate(data).CopyFrom(x.data))
}

// Stopped generates a stopped log event reporting that the activity
// or component the Task represents has paused or shutdown.  The
// variadic data parameter is aggregated as a D and reported with the
// event, as is the data permanently associated with the Task.  The
// given data is not associated to the Task permanently.
func (x *Task) Stopped(data ...interface{}) {
	x.root.event(x, STOPPED, x.activity+" stopped",
		DAggregate(data).CopyFrom(x.data))
}

// Finalized generates an end log event reporting that the component
// the Task represents has ceased.  It is generally intended to be
// used for components, while Success is used for discrete activities.
// Continuing to use the Task is discouraged.  The variadic data
// parameter is aggregated as a D and reported with the event, as is
// the data permanently associated with the Task.  The given data is
// not associated to the Task permanently.
func (x *Task) Finalized(data ...interface{}) {
	x.root.event(x, END, x.activity+" finalized", DAggregate(data).CopyFrom(x.data))
}

// Success generates a success log event reporting that the activity
// the Task represents has concluded successfully.  It always returns
// nil.  Continuing to use the Task is discouraged.  The variadic data
// parameter is aggregated as a D and reported with the event, as is
// the data permanently associated with the Task.  The given data is
// not associated to the Task permanently.
func (x *Task) Success(data ...interface{}) error {
	x.root.event(x, SUCCESS, x.activity+" success", DAggregate(data).CopyFrom(x.data))
	return nil
}

// Error generates an error log event reporting an unrecoverable fault
// in an activity or component.  An error is returned wrapping the
// original error with a message reporting that the Task's activity
// has failed.  Continuing to use the Task is discouraged.  The
// variadic data parameter is aggregated as a D and embedded in the
// generated error.  It and the data permanently associated with the
// Task is reported with the event.  The reported source code position
// of the generated task error is adjusted to be the event invocation.
func (x *Task) Error(err error, data ...interface{}) error {

	m := x.activity + " failed"

	e := wraperror(m, err, data)
	e.Locate(1)
	e.reported = true

	rep := D{
		"Source": e.Source,
	}

	if ex, ok := err.(*Error); !ok || !ex.reported {
		rep["Cause"] = DAggregate([]interface{}{err})
	}

	rep.CopyFrom(DAggregate(data))
	rep.CopyFrom(x.data)

	x.root.event(x, ERROR, m, rep)

	return e
}

func (x *Task) WrapError(msg string, source error, data ...interface{}) error {
	err := wraperror(msg, source, nil)
	err.Locate(1)

	m := x.activity + " failed"
	x.root.event(x, ERROR, m, D{"Error": DAggregate([]interface{}{err})}.CopyFrom(DAggregate(data)).CopyFrom(x.data))

	e := wraperror(m, err, data)
	e.Locate(1)
	return e
}

// Failure generates an error log event reporting an unrecoverable
// fault.  Failure and Error are essentially the same, the difference
// being that Failure is the first point of fault while Error takes an
// underlying error typically returned from another function or
// component.  An error is returned reporting that the activity or
// component represented by the Task has failed due to the underlying
// cause given in the message.  Continuing to use the Task is
// discouraged.  The variadic data parameter is aggregated as a D and
// embedded in the generated task error.  It and the data permanently
// associated with the Task is reported with the event.  The reported
// source code position of the generated task error is adjusted to be
// the event invocation.
func (x *Task) Failure(msg string, data ...interface{}) error {
	err := newerror(msg, nil)
	err.Locate(1)

	m := x.activity + " failed"
	x.root.event(x, ERROR, m, D{"Error": DAggregate([]interface{}{err})}.CopyFrom(DAggregate(data)).CopyFrom(x.data))

	e := wraperror(m, err, data)
	e.Locate(1)
	return e
}
<|endoftext|>


<commit_before>// Object and ObjectBase is an abstract class for all scheme expressions.
// A return value of a method which returns scheme object is Object.
// And ObjectBase has Object's implementation of String().

package scheme

type Object interface {
	Parent() Object
	Bounder() *Variable
	setParent(Object)
	setBounder(*Variable)
	Eval() Object
	String() string
	isNumber() bool
	isBoolean() bool
	isProcedure() bool
	isNull() bool
	isPair() bool
	isList() bool
	isSymbol() bool
	isSyntax() bool
	isString() bool
	isVariable() bool
	isApplication() bool
	define(string, Object)
	set(string, Object)
	scopedBinding() Binding
	binding() Binding
	boundedObject(string) Object
	ancestor() Object
}

type Binding map[string]Object

type ObjectBase struct {
	parent  Object
	bounder *Variable // Variable.Eval() sets itself into this
}

func (o *ObjectBase) Eval() Object {
	runtimeError("This object's Eval() is not implemented yet.")
	return nil
}

func (o *ObjectBase) String() string {
	runtimeError("This object's String() is not implemented yet.")
	return ""
}

func (o *ObjectBase) isNumber() bool {
	return false
}

func (o *ObjectBase) isBoolean() bool {
	return false
}

func (o *ObjectBase) isProcedure() bool {
	return false
}

func (o *ObjectBase) isNull() bool {
	return false
}

func (o *ObjectBase) isPair() bool {
	return false
}

func (o *ObjectBase) isList() bool {
	return false
}

func (o *ObjectBase) isSymbol() bool {
	return false
}

func (o *ObjectBase) isSyntax() bool {
	return false
}

func (o *ObjectBase) isString() bool {
	return false
}

func (o *ObjectBase) isVariable() bool {
	return false
}

func (o *ObjectBase) isApplication() bool {
	return false
}

func (o *ObjectBase) binding() Binding {
	return Binding{}
}

func (o *ObjectBase) Parent() Object {
	return o.parent
}

func (o *ObjectBase) Bounder() *Variable {
	return o.bounder
}

func (o *ObjectBase) setParent(parent Object) {
	o.parent = parent
}

func (o *ObjectBase) setBounder(bounder *Variable) {
	o.bounder = bounder
}

func (o *ObjectBase) scopedBinding() (scopedBinding Binding) {
	scopedBinding = make(Binding)
	parent := o.Parent()

	for parent != nil {
		for identifier, object := range parent.binding() {
			if scopedBinding[identifier] == nil {
				scopedBinding[identifier] = object
			}
		}
		parent = parent.Parent()
	}
	return
}

// Define variable in the most inner closure
func (o *ObjectBase) define(identifier string, object Object) {
	if o.parent == nil {
		runtimeError("Bind called for object whose parent is nil")
	} else {
		o.parent.define(identifier, object)
	}
}

// This is for set! syntax.
// Update the variable's value when it is defined.
func (o *ObjectBase) set(identifier string, object Object) {
	if o.parent == nil {
		runtimeError("symbol not defined")
	} else {
		o.parent.set(identifier, object)
	}
}

func (o *ObjectBase) boundedObject(identifier string) Object {
	scopedBinding := make(Binding)
	parent := o.Parent()

	for parent != nil {
		for identifier, object := range parent.binding() {
			if scopedBinding[identifier] == nil {
				scopedBinding[identifier] = object
			}
		}
		parent = parent.Parent()
	}

	return scopedBinding[identifier]
}

func (o *ObjectBase) ancestor() Object {
	ancestor := o.Parent()
	for {
		if ancestor.Parent() == nil {
			break
		} else {
			ancestor = ancestor.Parent()
		}
	}
	return ancestor
}
<commit_msg>Remove obsolete method<commit_after>// Object and ObjectBase is an abstract class for all scheme expressions.
// A return value of a method which returns scheme object is Object.
// And ObjectBase has Object's implementation of String().

package scheme

type Object interface {
	Parent() Object
	Bounder() *Variable
	setParent(Object)
	setBounder(*Variable)
	Eval() Object
	String() string
	isNumber() bool
	isBoolean() bool
	isProcedure() bool
	isNull() bool
	isPair() bool
	isList() bool
	isSymbol() bool
	isSyntax() bool
	isString() bool
	isVariable() bool
	isApplication() bool
	define(string, Object)
	set(string, Object)
	scopedBinding() Binding
	binding() Binding
	boundedObject(string) Object
}

type Binding map[string]Object

type ObjectBase struct {
	parent  Object
	bounder *Variable // Variable.Eval() sets itself into this
}

func (o *ObjectBase) Eval() Object {
	runtimeError("This object's Eval() is not implemented yet.")
	return nil
}

func (o *ObjectBase) String() string {
	runtimeError("This object's String() is not implemented yet.")
	return ""
}

func (o *ObjectBase) isNumber() bool {
	return false
}

func (o *ObjectBase) isBoolean() bool {
	return false
}

func (o *ObjectBase) isProcedure() bool {
	return false
}

func (o *ObjectBase) isNull() bool {
	return false
}

func (o *ObjectBase) isPair() bool {
	return false
}

func (o *ObjectBase) isList() bool {
	return false
}

func (o *ObjectBase) isSymbol() bool {
	return false
}

func (o *ObjectBase) isSyntax() bool {
	return false
}

func (o *ObjectBase) isString() bool {
	return false
}

func (o *ObjectBase) isVariable() bool {
	return false
}

func (o *ObjectBase) isApplication() bool {
	return false
}

func (o *ObjectBase) binding() Binding {
	return Binding{}
}

func (o *ObjectBase) Parent() Object {
	return o.parent
}

func (o *ObjectBase) Bounder() *Variable {
	return o.bounder
}

func (o *ObjectBase) setParent(parent Object) {
	o.parent = parent
}

func (o *ObjectBase) setBounder(bounder *Variable) {
	o.bounder = bounder
}

func (o *ObjectBase) scopedBinding() (scopedBinding Binding) {
	scopedBinding = make(Binding)
	parent := o.Parent()

	for parent != nil {
		for identifier, object := range parent.binding() {
			if scopedBinding[identifier] == nil {
				scopedBinding[identifier] = object
			}
		}
		parent = parent.Parent()
	}
	return
}

// Define variable in the most inner closure
func (o *ObjectBase) define(identifier string, object Object) {
	if o.parent == nil {
		runtimeError("Bind called for object whose parent is nil")
	} else {
		o.parent.define(identifier, object)
	}
}

// This is for set! syntax.
// Update the variable's value when it is defined.
func (o *ObjectBase) set(identifier string, object Object) {
	if o.parent == nil {
		runtimeError("symbol not defined")
	} else {
		o.parent.set(identifier, object)
	}
}

func (o *ObjectBase) boundedObject(identifier string) Object {
	scopedBinding := make(Binding)
	parent := o.Parent()

	for parent != nil {
		for identifier, object := range parent.binding() {
			if scopedBinding[identifier] == nil {
				scopedBinding[identifier] = object
			}
		}
		parent = parent.Parent()
	}

	return scopedBinding[identifier]
}
<|endoftext|>


<commit_before>package configmigrator

import (
	"io/ioutil"
	"os"
	"path"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/pivotal-cf/cf-redis-broker/redisconf"
)

var _ = Describe("Migrating config", func() {

	var configMigrator *ConfigMigrator
	var redisDataDirPath string

	BeforeEach(func() {
		var err error
		redisDataDirPath, err = ioutil.TempDir("", "redis-data")
		Expect(err).ToNot(HaveOccurred())

		configMigrator = &ConfigMigrator{
			RedisDataDir: redisDataDirPath,
		}
	})

	Context("when there is no data to migrate", func() {
		It("does nothing", func() {
			err := configMigrator.Migrate()
			Expect(err).ToNot(HaveOccurred())
		})
	})

	Context("when there is data to migrate", func() {
		var instanceBaseDir string

		BeforeEach(func() {
			instanceBaseDir = path.Join(redisDataDirPath, "instance1")
			err := os.Mkdir(instanceBaseDir, 0777)
			Expect(err).ToNot(HaveOccurred())
		})

		Context("and port is in redis-server.port and password is in redis-server.password", func() {
			It("deletes the redis port file and password file", func() {
				redisConfFile := path.Join(instanceBaseDir, "redis.conf")
				ioutil.WriteFile(redisConfFile, []byte("#port 63490"), 0777)

				redisPortFilePath := path.Join(instanceBaseDir, REDIS_PORT_FILENAME)
				ioutil.WriteFile(redisPortFilePath, []byte("3455"), 0777)
				redisPasswordFilePath := path.Join(instanceBaseDir, REDIS_PASSWORD_FILENAME)
				ioutil.WriteFile(redisPasswordFilePath, []byte("secret-password"), 0777)

				configMigrator.Migrate()

				_, err := os.Stat(redisPortFilePath)
				Expect(os.IsNotExist(err)).To(BeTrue())
				_, err = os.Stat(redisPasswordFilePath)
				Expect(os.IsNotExist(err)).To(BeTrue())
			})

			It("copies the port and password to redis.conf", func() {
				redisConfFile := path.Join(instanceBaseDir, "redis.conf")
				ioutil.WriteFile(redisConfFile, []byte("#port 63490"), 0777)

				redisPortFilePath := path.Join(instanceBaseDir, REDIS_PORT_FILENAME)
				ioutil.WriteFile(redisPortFilePath, []byte("3455"), 0777)

				redisPasswordFilePath := path.Join(instanceBaseDir, REDIS_PASSWORD_FILENAME)
				ioutil.WriteFile(redisPasswordFilePath, []byte("secret-password"), 0777)

				configMigrator.Migrate()

				redisConfigValues, err := redisconf.Load(redisConfFile)
				Expect(err).ToNot(HaveOccurred())
				Expect(redisConfigValues.Get("port")).To(Equal("3455"))
				Expect(redisConfigValues.Get("requirepass")).To(Equal("secret-password"))
			})

			It("does not change the other values", func() {
				redisConfFile := path.Join(instanceBaseDir, "redis.conf")
				ioutil.WriteFile(redisConfFile, []byte("foo bar"), 0777)

				configMigrator.Migrate()

				redisConfigValues, err := redisconf.Load(redisConfFile)
				Expect(err).ToNot(HaveOccurred())
				Expect(redisConfigValues.Get("foo")).To(Equal("bar"))
			})

			Context("and there are multiple instances to migrate", func() {
				It("migrates all of them", func() {
					redisConfFile := path.Join(instanceBaseDir, "redis.conf")
					ioutil.WriteFile(redisConfFile, []byte("#port 63490"), 0777)
					redisPortFilePath := path.Join(instanceBaseDir, REDIS_PORT_FILENAME)
					ioutil.WriteFile(redisPortFilePath, []byte("3455"), 0777)
					redisPasswordFilePath := path.Join(instanceBaseDir, REDIS_PASSWORD_FILENAME)
					ioutil.WriteFile(redisPasswordFilePath, []byte("secret-password"), 0777)

					instance2BaseDir := path.Join(redisDataDirPath, "instance2")
					os.Mkdir(instance2BaseDir, 0777)
					redis2ConfFile := path.Join(instance2BaseDir, "redis.conf")
					ioutil.WriteFile(redis2ConfFile, []byte("#port 63490"), 0777)
					redis2PortFilePath := path.Join(instance2BaseDir, REDIS_PORT_FILENAME)
					ioutil.WriteFile(redis2PortFilePath, []byte("9482"), 0777)
					redis2PasswordFilePath := path.Join(instance2BaseDir, REDIS_PASSWORD_FILENAME)
					ioutil.WriteFile(redis2PasswordFilePath, []byte("secret-password2"), 0777)

					configMigrator.Migrate()

					redisConfigValues, err := redisconf.Load(redisConfFile)
					Expect(err).ToNot(HaveOccurred())
					Expect(redisConfigValues.Get("port")).To(Equal("3455"))

					redisConfigValues, err = redisconf.Load(redis2ConfFile)
					Expect(err).ToNot(HaveOccurred())
					Expect(redisConfigValues.Get("port")).To(Equal("9482"))
				})
			})

			Context("and it cannot write to redis.conf", func() {
				It("returns an error", func() {
					redisConfFile := path.Join(instanceBaseDir, "redis.conf")
					ioutil.WriteFile(redisConfFile, []byte("foo bar"), 0000)

					err := configMigrator.Migrate()
					Expect(err).To(HaveOccurred())
				})
			})

			Context("and it cannot read from the redis-server.port", func() {
				It("returns an error", func() {
					redisConfFile := path.Join(instanceBaseDir, "redis.conf")
					ioutil.WriteFile(redisConfFile, []byte("#port 63490"), 0777)
					redisPortFilePath := path.Join(instanceBaseDir, REDIS_PORT_FILENAME)
					ioutil.WriteFile(redisPortFilePath, []byte("3455"), 0000)

					err := configMigrator.Migrate()
					Expect(err).To(HaveOccurred())
				})
			})
		})

		Context("and data is already migrated", func() {
			It("does nothing", func() {
				redisConfFile := path.Join(instanceBaseDir, "redis.conf")
				ioutil.WriteFile(redisConfFile, []byte("port 6349\nrequirepass secret-password"), 0777)

				err := configMigrator.Migrate()
				Expect(err).ToNot(HaveOccurred())

				redisConfigValues, err := redisconf.Load(path.Join(instanceBaseDir, "redis.conf"))
				Expect(err).ToNot(HaveOccurred())
				Expect(redisConfigValues.Get("port")).To(Equal("6349"))
				Expect(redisConfigValues.Get("requirepass")).To(Equal("secret-password"))
			})
		})

		Context("and data is partially migrated", func() {
			It("finishes the migration for the password", func() {
				redisConfFile := path.Join(instanceBaseDir, "redis.conf")
				ioutil.WriteFile(redisConfFile, []byte("port 6349\n#requirepass INSERT_PASSWORD_HERE"), 0777)
				redisPasswordFilePath := path.Join(instanceBaseDir, REDIS_PASSWORD_FILENAME)
				ioutil.WriteFile(redisPasswordFilePath, []byte("secret-password"), 0777)

				err := configMigrator.Migrate()
				Expect(err).ToNot(HaveOccurred())

				redisConfigValues, err := redisconf.Load(path.Join(instanceBaseDir, "redis.conf"))
				Expect(err).ToNot(HaveOccurred())
				Expect(redisConfigValues.Get("port")).To(Equal("6349"))
				Expect(redisConfigValues.Get("requirepass")).To(Equal("secret-password"))
			})
			It("finishes the migration for the port", func() {
				redisConfFile := path.Join(instanceBaseDir, "redis.conf")
				ioutil.WriteFile(redisConfFile, []byte("#port 1234\nrequirepass secret-password"), 0777)
				redisPortFilePath := path.Join(instanceBaseDir, REDIS_PORT_FILENAME)
				ioutil.WriteFile(redisPortFilePath, []byte("6349"), 0777)

				err := configMigrator.Migrate()
				Expect(err).ToNot(HaveOccurred())

				redisConfigValues, err := redisconf.Load(path.Join(instanceBaseDir, "redis.conf"))
				Expect(err).ToNot(HaveOccurred())
				Expect(redisConfigValues.Get("port")).To(Equal("6349"))
				Expect(redisConfigValues.Get("requirepass")).To(Equal("secret-password"))
			})
		})

		Context("and loading of the redis.conf file is failing", func() {
			It("returns a error", func() {
				err := configMigrator.Migrate()

				Expect(err).To(HaveOccurred())
			})
		})
	})
})
<commit_msg>Moving definitions to before each to improve readability<commit_after>package configmigrator

import (
	"io/ioutil"
	"os"
	"path"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/pivotal-cf/cf-redis-broker/redisconf"
)

var _ = Describe("Migrating config", func() {

	var configMigrator *ConfigMigrator
	var redisDataDirPath string
	var redisConfFile string
	var instanceBaseDir string
	var redisPortFilePath string
	var redisPasswordFilePath string

	BeforeEach(func() {
		var err error
		redisDataDirPath, err = ioutil.TempDir("", "redis-data")
		Expect(err).ToNot(HaveOccurred())

		configMigrator = &ConfigMigrator{
			RedisDataDir: redisDataDirPath,
		}
	})

	Context("when there is no data to migrate", func() {
		It("does nothing", func() {
			err := configMigrator.Migrate()
			Expect(err).ToNot(HaveOccurred())
		})
	})

	Context("when there is data to migrate", func() {
		BeforeEach(func() {
			instanceBaseDir = path.Join(redisDataDirPath, "instance1")
			os.Mkdir(instanceBaseDir, 0777)
			redisConfFile = path.Join(instanceBaseDir, "redis.conf")
			redisPortFilePath = path.Join(instanceBaseDir, REDIS_PORT_FILENAME)
			redisPasswordFilePath = path.Join(instanceBaseDir, REDIS_PASSWORD_FILENAME)
		})

		Context("and port is in redis-server.port and password is in redis-server.password", func() {
			It("deletes the redis port file and password file", func() {
				ioutil.WriteFile(redisConfFile, []byte("#port 63490"), 0777)
				ioutil.WriteFile(redisPortFilePath, []byte("3455"), 0777)
				ioutil.WriteFile(redisPasswordFilePath, []byte("secret-password"), 0777)

				configMigrator.Migrate()

				_, err := os.Stat(redisPortFilePath)
				Expect(os.IsNotExist(err)).To(BeTrue())
				_, err = os.Stat(redisPasswordFilePath)
				Expect(os.IsNotExist(err)).To(BeTrue())
			})

			It("copies the port and password to redis.conf", func() {
				ioutil.WriteFile(redisConfFile, []byte("#port 63490"), 0777)
				ioutil.WriteFile(redisPortFilePath, []byte("3455"), 0777)
				ioutil.WriteFile(redisPasswordFilePath, []byte("secret-password"), 0777)

				configMigrator.Migrate()

				redisConfigValues, err := redisconf.Load(redisConfFile)
				Expect(err).ToNot(HaveOccurred())
				Expect(redisConfigValues.Get("port")).To(Equal("3455"))
				Expect(redisConfigValues.Get("requirepass")).To(Equal("secret-password"))
			})

			It("does not change the other values", func() {
				ioutil.WriteFile(redisConfFile, []byte("foo bar"), 0777)

				configMigrator.Migrate()

				redisConfigValues, err := redisconf.Load(redisConfFile)
				Expect(err).ToNot(HaveOccurred())
				Expect(redisConfigValues.Get("foo")).To(Equal("bar"))
			})

			Context("and there are multiple instances to migrate", func() {
				It("migrates all of them", func() {
					ioutil.WriteFile(redisConfFile, []byte("#port 63490"), 0777)
					ioutil.WriteFile(redisPortFilePath, []byte("3455"), 0777)
					ioutil.WriteFile(redisPasswordFilePath, []byte("secret-password"), 0777)

					instance2BaseDir := path.Join(redisDataDirPath, "instance2")
					os.Mkdir(instance2BaseDir, 0777)
					redis2ConfFile := path.Join(instance2BaseDir, "redis.conf")
					ioutil.WriteFile(redis2ConfFile, []byte("#port 63490"), 0777)
					redis2PortFilePath := path.Join(instance2BaseDir, REDIS_PORT_FILENAME)
					ioutil.WriteFile(redis2PortFilePath, []byte("9482"), 0777)
					redis2PasswordFilePath := path.Join(instance2BaseDir, REDIS_PASSWORD_FILENAME)
					ioutil.WriteFile(redis2PasswordFilePath, []byte("secret-password2"), 0777)

					configMigrator.Migrate()

					redisConfigValues, err := redisconf.Load(redisConfFile)
					Expect(err).ToNot(HaveOccurred())
					Expect(redisConfigValues.Get("port")).To(Equal("3455"))

					redisConfigValues, err = redisconf.Load(redis2ConfFile)
					Expect(err).ToNot(HaveOccurred())
					Expect(redisConfigValues.Get("port")).To(Equal("9482"))
				})
			})

			Context("and it cannot write to redis.conf", func() {
				It("returns an error", func() {
					ioutil.WriteFile(redisConfFile, []byte("foo bar"), 0000)

					err := configMigrator.Migrate()
					Expect(err).To(HaveOccurred())
				})
			})

			Context("and it cannot read from the redis-server.port", func() {
				It("returns an error", func() {
					ioutil.WriteFile(redisConfFile, []byte("#port 63490"), 0777)
					ioutil.WriteFile(redisPortFilePath, []byte("3455"), 0000)

					err := configMigrator.Migrate()
					Expect(err).To(HaveOccurred())
				})
			})
		})

		Context("and data is already migrated", func() {
			It("does nothing", func() {
				ioutil.WriteFile(redisConfFile, []byte("port 6349\nrequirepass secret-password"), 0777)

				err := configMigrator.Migrate()
				Expect(err).ToNot(HaveOccurred())

				redisConfigValues, err := redisconf.Load(path.Join(instanceBaseDir, "redis.conf"))
				Expect(err).ToNot(HaveOccurred())
				Expect(redisConfigValues.Get("port")).To(Equal("6349"))
				Expect(redisConfigValues.Get("requirepass")).To(Equal("secret-password"))
			})
		})

		Context("and data is partially migrated", func() {
			It("finishes the migration for the password", func() {
				ioutil.WriteFile(redisConfFile, []byte("port 6349\n#requirepass INSERT_PASSWORD_HERE"), 0777)
				ioutil.WriteFile(redisPasswordFilePath, []byte("secret-password"), 0777)

				err := configMigrator.Migrate()
				Expect(err).ToNot(HaveOccurred())

				redisConfigValues, err := redisconf.Load(path.Join(instanceBaseDir, "redis.conf"))
				Expect(err).ToNot(HaveOccurred())
				Expect(redisConfigValues.Get("port")).To(Equal("6349"))
				Expect(redisConfigValues.Get("requirepass")).To(Equal("secret-password"))
			})

			It("finishes the migration for the port", func() {
				ioutil.WriteFile(redisConfFile, []byte("#port 1234\nrequirepass secret-password"), 0777)
				ioutil.WriteFile(redisPortFilePath, []byte("6349"), 0777)

				err := configMigrator.Migrate()
				Expect(err).ToNot(HaveOccurred())

				redisConfigValues, err := redisconf.Load(path.Join(instanceBaseDir, "redis.conf"))
				Expect(err).ToNot(HaveOccurred())
				Expect(redisConfigValues.Get("port")).To(Equal("6349"))
				Expect(redisConfigValues.Get("requirepass")).To(Equal("secret-password"))
			})
		})

		Context("and loading of the redis.conf file is failing", func() {
			It("returns a error", func() {
				err := configMigrator.Migrate()

				Expect(err).To(HaveOccurred())
			})
		})
	})
})
<|endoftext|>


<commit_before>package keeper

import (
	"context"

	sdk "github.com/cosmos/cosmos-sdk/types"
	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
	"github.com/zigbee-alliance/distributed-compliance-ledger/x/validator/types"
)

func (k msgServer) ApproveDisableValidator(goCtx context.Context, msg *types.MsgApproveDisableValidator) (*types.MsgApproveDisableValidatorResponse, error) {
	ctx := sdk.UnwrapSDKContext(goCtx)

	creatorAddr, err := sdk.AccAddressFromBech32(msg.Creator)
	if err != nil {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrInvalidAddress, "Invalid creator address: (%s)", err)
	}

	validatorAddr, err := sdk.ValAddressFromBech32(msg.Address)
	if err != nil {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrInvalidAddress, "Invalid validator address: (%s)", err)
	}

	// check if message creator has enough rights to approve disable validator
	if !k.dclauthKeeper.HasRole(ctx, creatorAddr, types.VoteForDisableValidatorRole) {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrUnauthorized,
			"MsgApproveDisableValidator transaction should be signed by an account with the %s role",
			types.VoteForDisableValidatorRole,
		)
	}

	// check if proposed disable validator exists
	proposedDisableValidator, isFound := k.GetProposedDisableValidator(ctx, validatorAddr.String())
	if !isFound {
		return nil, types.NewErrProposedDisableValidatorDoesNotExist(msg.Address)
	}

	// check if disable validator already has reject from message creator
	if proposedDisableValidator.HasRejectDisableFrom(creatorAddr) {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrUnauthorized,
			"Disabled validator with address=%v already has reject from=%v",
			msg.Address,
			msg.Creator,
		)
	}

	// check if disable validator already has approval form message creator
	if proposedDisableValidator.HasApprovalFrom(creatorAddr) {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrUnauthorized,
			"Disabled validator with address=%v already has approval from=%v",
			msg.Address, msg.Creator,
		)
	}

	// append approval
	grant := types.Grant{
		Address: creatorAddr.String(),
		Time:    msg.Time,
		Info:    msg.Info,
	}

	proposedDisableValidator.Approvals = append(proposedDisableValidator.Approvals, &grant)

	// check if proposed disable validator has enough approvals
	if len(proposedDisableValidator.Approvals) == k.DisableValidatorApprovalsCount(ctx) {
		// remove disable validator
		k.RemoveProposedDisableValidator(ctx, proposedDisableValidator.Address)

		approvedDisableValidator := types.DisabledValidator{
			Address:             proposedDisableValidator.Address,
			Creator:             proposedDisableValidator.Creator,
			Approvals:           proposedDisableValidator.Approvals,
			RejectApprovals:     proposedDisableValidator.RejectApprovals,
			DisabledByNodeAdmin: false,
		}

		// Disable validator
		validator, _ := k.GetValidator(ctx, validatorAddr)
		k.Jail(ctx, validator, proposedDisableValidator.Approvals[0].Info)

		k.SetDisabledValidator(ctx, approvedDisableValidator)
	} else {
		// update proposed disable validator
		k.SetProposedDisableValidator(ctx, proposedDisableValidator)
	}

	return &types.MsgApproveDisableValidatorResponse{}, nil
}
<commit_msg>Add implementation in approve disable validator: if we try to re-adding rejected account we should remove from the entity rejectedDisableAccount<commit_after>package keeper

import (
	"context"

	sdk "github.com/cosmos/cosmos-sdk/types"
	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
	"github.com/zigbee-alliance/distributed-compliance-ledger/x/validator/types"
)

func (k msgServer) ApproveDisableValidator(goCtx context.Context, msg *types.MsgApproveDisableValidator) (*types.MsgApproveDisableValidatorResponse, error) {
	ctx := sdk.UnwrapSDKContext(goCtx)

	creatorAddr, err := sdk.AccAddressFromBech32(msg.Creator)
	if err != nil {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrInvalidAddress, "Invalid creator address: (%s)", err)
	}

	validatorAddr, err := sdk.ValAddressFromBech32(msg.Address)
	if err != nil {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrInvalidAddress, "Invalid validator address: (%s)", err)
	}

	// check if message creator has enough rights to approve disable validator
	if !k.dclauthKeeper.HasRole(ctx, creatorAddr, types.VoteForDisableValidatorRole) {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrUnauthorized,
			"MsgApproveDisableValidator transaction should be signed by an account with the %s role",
			types.VoteForDisableValidatorRole,
		)
	}

	// check if proposed disable validator exists
	proposedDisableValidator, isFound := k.GetProposedDisableValidator(ctx, validatorAddr.String())
	if !isFound {
		return nil, types.NewErrProposedDisableValidatorDoesNotExist(msg.Address)
	}

	// check if disable validator already has reject from message creator
	if proposedDisableValidator.HasRejectDisableFrom(creatorAddr) {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrUnauthorized,
			"Disabled validator with address=%v already has reject from=%v",
			msg.Address,
			msg.Creator,
		)
	}

	// check if disable validator already has approval form message creator
	if proposedDisableValidator.HasApprovalFrom(creatorAddr) {
		return nil, sdkerrors.Wrapf(sdkerrors.ErrUnauthorized,
			"Disabled validator with address=%v already has approval from=%v",
			msg.Address, msg.Creator,
		)
	}

	// append approval
	grant := types.Grant{
		Address: creatorAddr.String(),
		Time:    msg.Time,
		Info:    msg.Info,
	}

	proposedDisableValidator.Approvals = append(proposedDisableValidator.Approvals, &grant)

	// check if proposed disable validator has enough approvals
	if len(proposedDisableValidator.Approvals) == k.DisableValidatorApprovalsCount(ctx) {
		_, isFound := k.GetRejectedNode(ctx, proposedDisableValidator.Address)
		if isFound {
			k.RemoveRejectedNode(ctx, proposedDisableValidator.Address)
		}

		// remove disable validator
		k.RemoveProposedDisableValidator(ctx, proposedDisableValidator.Address)

		approvedDisableValidator := types.DisabledValidator{
			Address:             proposedDisableValidator.Address,
			Creator:             proposedDisableValidator.Creator,
			Approvals:           proposedDisableValidator.Approvals,
			RejectApprovals:     proposedDisableValidator.RejectApprovals,
			DisabledByNodeAdmin: false,
		}

		// Disable validator
		validator, _ := k.GetValidator(ctx, validatorAddr)
		k.Jail(ctx, validator, proposedDisableValidator.Approvals[0].Info)

		k.SetDisabledValidator(ctx, approvedDisableValidator)
	} else {
		// update proposed disable validator
		k.SetProposedDisableValidator(ctx, proposedDisableValidator)
	}

	return &types.MsgApproveDisableValidatorResponse{}, nil
}
<|endoftext|>


<commit_before>package osdconfig

import (
	"context"
	"encoding/json"
	"fmt"
	"testing"
	"time"

	"github.com/pkg/errors"
)

func TestWatch(t *testing.T) {
	// create in memory kvdb
	kv, err := newInMemKvdb()
	if err != nil {
		t.Fatal(err)
	}

	// create new manager
	manager, err := NewManager(context.Background(), kv)
	if err != nil {
		t.Fatal(err)
	}
	defer manager.Close()

	// register cluster watcher callback
	if err := manager.WatchCluster("clusterWatcher", clusterWatcher); err != nil {
		t.Fatal(err)
	}

	// register node watcher callback
	if err := manager.WatchNode("nodeWatcher", nodeWatcher); err != nil {
		t.Fatal(err)
	}

	// update a few values
	if err := setSomeClusterValues(manager); err != nil {
		t.Fatal(err)
	}

	// update more values... each of these updates will trigger callback execution
	if err := setSomeNodeValues(manager); err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Second)

}

// clusterWatcher is an example callback function to watch on cluster config changes
func clusterWatcher(config *ClusterConfig) error {
	if jb, err := json.MarshalIndent(config, "", "  "); err != nil {
		return err
	} else {
		fmt.Println(string(jb))
		if config.ClusterId != "myClusterID" {
			return errors.New("expected myClusterID, received " + config.ClusterId)
			panic(DataErr)
		}
		if config.Driver != "myDriver" {
			return errors.New("expected myDriver, receive " + config.Driver)
			panic(DataErr)
		}
	}
	return nil
}

// nodeWatcher is an example callback function to watch on node config changes
func nodeWatcher(config *NodeConfig) error {
	if jb, err := json.MarshalIndent(config, "", "  "); err != nil {
		return err
	} else {
		fmt.Println(string(jb))
		if config.Network.DataIface != "dataIface" {
			return errors.New("expected dataIface, received " + config.Network.DataIface)
			panic(DataErr)
		}
	}
	return nil
}

// setSomeClusterValues is a helper function to set cluster config values in kvdb
func setSomeClusterValues(manager ConfigManager) error {
	// prepare expected cluster config
	conf := new(ClusterConfig)
	conf.ClusterId = "myClusterID"
	conf.Driver = "myDriver"

	if err := manager.SetClusterConf(conf); err != nil {
		return err
	}

	return nil
}

// setSomeNodeValues is a helper function to set some node config values in kvdb
func setSomeNodeValues(manager ConfigManager) error {
	// prepare expected cluster config
	conf := new(NodesConfig)
	conf.NodeConf = make(map[string]*NodeConfig)
	conf.NodeConf["node1"] = new(NodeConfig)
	conf.NodeConf["node2"] = new(NodeConfig)
	conf.NodeConf["node3"] = new(NodeConfig)

	for key, val := range conf.NodeConf {
		key, val := key, val
		val.NodeId = key
		val.Network = new(NetworkConfig)
		val.Network.DataIface = "dataIface"
		if err := manager.SetNodeConf(val); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>removed panic from tests<commit_after>package osdconfig

import (
	"context"
	"encoding/json"
	"fmt"
	"testing"
	"time"

	"github.com/pkg/errors"
)

func TestWatch(t *testing.T) {
	// create in memory kvdb
	kv, err := newInMemKvdb()
	if err != nil {
		t.Fatal(err)
	}

	// create new manager
	manager, err := NewManager(context.Background(), kv)
	if err != nil {
		t.Fatal(err)
	}
	defer manager.Close()

	// register cluster watcher callback
	if err := manager.WatchCluster("clusterWatcher", clusterWatcher); err != nil {
		t.Fatal(err)
	}

	// register node watcher callback
	if err := manager.WatchNode("nodeWatcher", nodeWatcher); err != nil {
		t.Fatal(err)
	}

	// update a few values
	if err := setSomeClusterValues(manager); err != nil {
		t.Fatal(err)
	}

	// update more values... each of these updates will trigger callback execution
	if err := setSomeNodeValues(manager); err != nil {
		t.Fatal(err)
	}

	time.Sleep(time.Second)

}

// clusterWatcher is an example callback function to watch on cluster config changes
func clusterWatcher(config *ClusterConfig) error {
	if jb, err := json.MarshalIndent(config, "", "  "); err != nil {
		return err
	} else {
		fmt.Println(string(jb))
		if config.ClusterId != "myClusterID" {
			return errors.New("expected myClusterID, received " + config.ClusterId)
			//panic(DataErr)
		}
		if config.Driver != "myDriver" {
			return errors.New("expected myDriver, receive " + config.Driver)
			//panic(DataErr)
		}
	}
	return nil
}

// nodeWatcher is an example callback function to watch on node config changes
func nodeWatcher(config *NodeConfig) error {
	if jb, err := json.MarshalIndent(config, "", "  "); err != nil {
		return err
	} else {
		fmt.Println(string(jb))
		if config.Network.DataIface != "dataIface" {
			return errors.New("expected dataIface, received " + config.Network.DataIface)
			panic(DataErr)
		}
	}
	return nil
}

// setSomeClusterValues is a helper function to set cluster config values in kvdb
func setSomeClusterValues(manager ConfigManager) error {
	// prepare expected cluster config
	conf := new(ClusterConfig)
	conf.ClusterId = "myClusterID"
	conf.Driver = "myDriver"

	if err := manager.SetClusterConf(conf); err != nil {
		return err
	}

	return nil
}

// setSomeNodeValues is a helper function to set some node config values in kvdb
func setSomeNodeValues(manager ConfigManager) error {
	// prepare expected cluster config
	conf := new(NodesConfig)
	conf.NodeConf = make(map[string]*NodeConfig)
	conf.NodeConf["node1"] = new(NodeConfig)
	conf.NodeConf["node2"] = new(NodeConfig)
	conf.NodeConf["node3"] = new(NodeConfig)

	for key, val := range conf.NodeConf {
		key, val := key, val
		val.NodeId = key
		val.Network = new(NetworkConfig)
		val.Network.DataIface = "dataIface"
		if err := manager.SetNodeConf(val); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package stream

import (
	"context"
	"errors"
	"fmt"
	"net"

	ss "github.com/libp2p/go-conn-security"
	pnet "github.com/libp2p/go-libp2p-interface-pnet"
	peer "github.com/libp2p/go-libp2p-peer"
	transport "github.com/libp2p/go-libp2p-transport"
	filter "github.com/libp2p/go-maddr-filter"
	smux "github.com/libp2p/go-stream-muxer"
	manet "github.com/multiformats/go-multiaddr-net"
)

// ErrNilPeer is returned when attempting to upgrade an outbound connection
// without specifying a peer ID.
var ErrNilPeer = errors.New("nil peer")

// AcceptQueueLength is the number of connections to fully setup before not accepting any new connections
var AcceptQueueLength = 16

// Upgrader is a multistream upgrader that can upgrade an underlying connection
// to a full transport connection (secure and multiplexed).
type Upgrader struct {
	Protector pnet.Protector
	Secure    ss.Transport
	Muxer     smux.Transport
	Filters   *filter.Filters
}

// UpgradeListener upgrades the passed multiaddr-net listener into a full libp2p-transport listener.
func (u *Upgrader) UpgradeListener(t transport.Transport, list manet.Listener) transport.Listener {
	ctx, cancel := context.WithCancel(context.Background())
	l := &listener{
		Listener:  list,
		upgrader:  u,
		transport: t,
		threshold: newThreshold(AcceptQueueLength),
		incoming:  make(chan transport.Conn),
		cancel:    cancel,
		ctx:       ctx,
	}
	go l.handleIncoming()
	return l
}

// UpgradeOutbound upgrades the given outbound multiaddr-net connection into a
// full libp2p-transport connection.
func (u *Upgrader) UpgradeOutbound(ctx context.Context, t transport.Transport, maconn manet.Conn, p peer.ID) (transport.Conn, error) {
	if p == "" {
		return nil, ErrNilPeer
	}
	return u.upgrade(ctx, t, maconn, p)
}

// UpgradeInbound upgrades the given inbound multiaddr-net connection into a
// full libp2p-transport connection.
func (u *Upgrader) UpgradeInbound(ctx context.Context, t transport.Transport, maconn manet.Conn) (transport.Conn, error) {
	return u.upgrade(ctx, t, maconn, "")
}

func (u *Upgrader) upgrade(ctx context.Context, t transport.Transport, maconn manet.Conn, p peer.ID) (transport.Conn, error) {
	if u.Filters != nil && u.Filters.AddrBlocked(maconn.RemoteMultiaddr()) {
		log.Debugf("blocked connection from %s", maconn.RemoteMultiaddr())
		maconn.Close()
		return nil, fmt.Errorf("blocked connection from %s", maconn.RemoteMultiaddr())
	}

	var conn net.Conn = maconn
	if u.Protector != nil {
		pconn, err := u.Protector.Protect(conn)
		if err != nil {
			conn.Close()
			return nil, fmt.Errorf("failed to setup private network protector: %s", err)
		}
		conn = pconn
	} else if pnet.ForcePrivateNetwork {
		log.Error("tried to dial with no Private Network Protector but usage" +
			" of Private Networks is forced by the enviroment")
		return nil, pnet.ErrNotInPrivateNetwork
	}
	sconn, err := u.setupSecurity(ctx, conn, p)
	if err != nil {
		conn.Close()
		return nil, fmt.Errorf("failed to negotiate security protocol: %s", err)
	}
	smconn, err := u.setupMuxer(ctx, sconn, p)
	if err != nil {
		conn.Close()
		return nil, fmt.Errorf("failed to negotiate security stream multiplexer: %s", err)
	}
	return &transportConn{
		Conn:           smconn,
		ConnMultiaddrs: maconn,
		ConnSecurity:   sconn,
		transport:      t,
	}, nil
}

func (u *Upgrader) setupSecurity(ctx context.Context, conn net.Conn, p peer.ID) (ss.Conn, error) {
	if p == "" {
		return u.Secure.SecureInbound(ctx, conn)
	}
	return u.Secure.SecureOutbound(ctx, conn, p)
}

func (u *Upgrader) setupMuxer(ctx context.Context, conn net.Conn, p peer.ID) (smux.Conn, error) {
	// TODO: The muxer should take a context.
	done := make(chan struct{})

	var smconn smux.Conn
	var err error
	go func() {
		defer close(done)
		smconn, err = u.Muxer.NewConn(conn, p == "")
	}()

	select {
	case <-done:
		return smconn, err
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}
<commit_msg>improve correctness of closing connections on failure<commit_after>package stream

import (
	"context"
	"errors"
	"fmt"
	"net"

	ss "github.com/libp2p/go-conn-security"
	pnet "github.com/libp2p/go-libp2p-interface-pnet"
	peer "github.com/libp2p/go-libp2p-peer"
	transport "github.com/libp2p/go-libp2p-transport"
	filter "github.com/libp2p/go-maddr-filter"
	smux "github.com/libp2p/go-stream-muxer"
	manet "github.com/multiformats/go-multiaddr-net"
)

// ErrNilPeer is returned when attempting to upgrade an outbound connection
// without specifying a peer ID.
var ErrNilPeer = errors.New("nil peer")

// AcceptQueueLength is the number of connections to fully setup before not accepting any new connections
var AcceptQueueLength = 16

// Upgrader is a multistream upgrader that can upgrade an underlying connection
// to a full transport connection (secure and multiplexed).
type Upgrader struct {
	Protector pnet.Protector
	Secure    ss.Transport
	Muxer     smux.Transport
	Filters   *filter.Filters
}

// UpgradeListener upgrades the passed multiaddr-net listener into a full libp2p-transport listener.
func (u *Upgrader) UpgradeListener(t transport.Transport, list manet.Listener) transport.Listener {
	ctx, cancel := context.WithCancel(context.Background())
	l := &listener{
		Listener:  list,
		upgrader:  u,
		transport: t,
		threshold: newThreshold(AcceptQueueLength),
		incoming:  make(chan transport.Conn),
		cancel:    cancel,
		ctx:       ctx,
	}
	go l.handleIncoming()
	return l
}

// UpgradeOutbound upgrades the given outbound multiaddr-net connection into a
// full libp2p-transport connection.
func (u *Upgrader) UpgradeOutbound(ctx context.Context, t transport.Transport, maconn manet.Conn, p peer.ID) (transport.Conn, error) {
	if p == "" {
		return nil, ErrNilPeer
	}
	return u.upgrade(ctx, t, maconn, p)
}

// UpgradeInbound upgrades the given inbound multiaddr-net connection into a
// full libp2p-transport connection.
func (u *Upgrader) UpgradeInbound(ctx context.Context, t transport.Transport, maconn manet.Conn) (transport.Conn, error) {
	return u.upgrade(ctx, t, maconn, "")
}

func (u *Upgrader) upgrade(ctx context.Context, t transport.Transport, maconn manet.Conn, p peer.ID) (transport.Conn, error) {
	if u.Filters != nil && u.Filters.AddrBlocked(maconn.RemoteMultiaddr()) {
		log.Debugf("blocked connection from %s", maconn.RemoteMultiaddr())
		maconn.Close()
		return nil, fmt.Errorf("blocked connection from %s", maconn.RemoteMultiaddr())
	}

	var conn net.Conn = maconn
	if u.Protector != nil {
		pconn, err := u.Protector.Protect(conn)
		if err != nil {
			conn.Close()
			return nil, fmt.Errorf("failed to setup private network protector: %s", err)
		}
		conn = pconn
	} else if pnet.ForcePrivateNetwork {
		log.Error("tried to dial with no Private Network Protector but usage" +
			" of Private Networks is forced by the enviroment")
		return nil, pnet.ErrNotInPrivateNetwork
	}
	sconn, err := u.setupSecurity(ctx, conn, p)
	if err != nil {
		conn.Close()
		return nil, fmt.Errorf("failed to negotiate security protocol: %s", err)
	}
	smconn, err := u.setupMuxer(ctx, sconn, p)
	if err != nil {
		sconn.Close()
		return nil, fmt.Errorf("failed to negotiate security stream multiplexer: %s", err)
	}
	return &transportConn{
		Conn:           smconn,
		ConnMultiaddrs: maconn,
		ConnSecurity:   sconn,
		transport:      t,
	}, nil
}

func (u *Upgrader) setupSecurity(ctx context.Context, conn net.Conn, p peer.ID) (ss.Conn, error) {
	if p == "" {
		return u.Secure.SecureInbound(ctx, conn)
	}
	return u.Secure.SecureOutbound(ctx, conn, p)
}

func (u *Upgrader) setupMuxer(ctx context.Context, conn net.Conn, p peer.ID) (smux.Conn, error) {
	// TODO: The muxer should take a context.
	done := make(chan struct{})

	var smconn smux.Conn
	var err error
	go func() {
		defer close(done)
		smconn, err = u.Muxer.NewConn(conn, p == "")
	}()

	select {
	case <-done:
		return smconn, err
	case <-ctx.Done():
		// interrupt this process
		conn.Close()
		// wait to finish
		<-done
		return nil, ctx.Err()
	}
}
<|endoftext|>


<commit_before>package aeds

import (
	"bytes"
	"encoding/gob"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/memcache"
)

// interface for structures that can be stored in App Engine's datastore
type Entity interface {
	Kind() string
	StringId() string
}

// HasGetHook is implemented by any Entity that wants to execute
// specific code after fetching the raw entity from datastore.
// This is often used to calculate derived fields.
type HasGetHook interface {
	HookAfterGet()
}

// HasPutHook is implemented by any Entity that wants to execute
// specific code before writing the raw entity to datastore.
// This is often used to calculate derived fields.
type HasPutHook interface {
	HookBeforePut()
}

// CanBeCached is implemented by any Entity that wants to
// have its values stored in memcache to improve read performance.
type CanBeCached interface {
	// CacheTtl indicates how long the entity should be cached in memcache.
	// Return zero to disable memcache.  If this method returns a non-zero
	// duration, the receiver should also implement the GobEncoder and
	// GobDecoder interfaces.
	CacheTtl() time.Duration
}

// NeedsIdempotentReset is implemented by any Entity that needs to reset its
// fields to zero when performing datastore read operations that are intended to
// be idempotent. For example, a datastore read into a slice property is not
// idempotent (it just appends property values to the slice).  In that case,
// this method might manually reset the slice length to 0 so that append leaves
// only the desired data.
type NeedsIdempotentReset interface {
	IdempotentReset()
}

// Key returns a datastore key for this entity.
func Key(c appengine.Context, e Entity) *datastore.Key {
	return datastore.NewKey(c, e.Kind(), e.StringId(), 0, nil)
}

// Put stores an entity in the datastore.
func Put(c appengine.Context, e Entity) (*datastore.Key, error) {
	if x, ok := e.(HasPutHook); ok {
		x.HookBeforePut()
	}

	// store entity in the datastore
	lookupKey := Key(c, e)
	key, err := datastore.Put(c, lookupKey, e)
	if err != nil {
		return nil, err
	}

	// delete from memcache?
	err = ClearCache(c, e)
	if err != nil {
		c.Errorf("aeds.Put ClearCache error: %s", err)
	}

	return key, nil
}

// ClearCache explicitly clears any memcache entries associated with this
// entity. One doesn't usually call this function directly.  Rather, it's called
// implicitly when other aeds functions know the cache should be cleared.
func ClearCache(c appengine.Context, e Entity) error {
	// nothing to do for uncacheable entities
	if !canBeCached(e) {
		return nil
	}

	err := memcache.Delete(c, Key(c, e).String())
	switch err {
	case nil:
	case memcache.ErrCacheMiss:
	default:
		return err
	}

	return nil
}

// Delete removes an entity from the datastore.
func Delete(c appengine.Context, e Entity) error {
	lookupKey := Key(c, e)

	// should the entity be removed from memcache too?
	err := ClearCache(c, e)
	if err != nil {
		return err
	}

	return datastore.Delete(c, lookupKey)
}

// FromId fetches an entity based on its ID.  The given entity
// should have enough data to calculate the entity's key.  On
// success, the entity is modified in place with all data from
// the datastore.
// Field mismatch errors are ignored.
func FromId(c appengine.Context, e Entity) (Entity, error) {
	lookupKey := Key(c, e)
	var ttl time.Duration
	if x, ok := e.(CanBeCached); ok {
		ttl = x.CacheTtl()
	}

	// should we look in memcache too?
	cacheMiss := false
	if ttl > 0 {
		item, err := memcache.Get(c, lookupKey.String())
		if err == nil {
			buf := bytes.NewBuffer(item.Value)
			err := gob.NewDecoder(buf).Decode(e)
			if x, ok := e.(HasGetHook); ok {
				x.HookAfterGet()
			}
			return e, err
		}
		if err == memcache.ErrCacheMiss {
			cacheMiss = true
		}
		// ignore any memcache errors
	}

	// look in the datastore
	err := datastore.Get(c, lookupKey, e)
	if err == nil || IsErrFieldMismatch(err) {
		if x, ok := e.(HasGetHook); ok {
			x.HookAfterGet()
		}

		// should we update memcache?
		if cacheMiss && ttl > 0 {
			if x, ok := e.(HasPutHook); ok {
				x.HookBeforePut()
			}

			// encode
			var value bytes.Buffer
			err := gob.NewEncoder(&value).Encode(e)
			if err != nil {
				return nil, err
			}

			// store
			item := &memcache.Item{
				Key:        lookupKey.String(),
				Value:      value.Bytes(),
				Expiration: ttl,
			}
			err = memcache.Set(c, item)
			_ = err // ignore memcache errors
		}

		return e, nil
	}
	return nil, err // unknown datastore error
}

// Modify atomically executes a read, modify, write operation on a single
// entity.  It should be used any time the results of a datastore read influence
// the contents of a datastore write.  Before executing f, the contents of e
// will be overwritten with the latest data available from the datastore.
//
// f should return an error value if something goes wrong with the modification.
// Modify returns that error value.
//
// As always, hooks defined by HookAfterGet() and HookBeforePut() are
// automatically executed at the appropriate time.  Be sure to define
// IdempotentReset() if your entity has any slice properties.
func Modify(c appengine.Context, e Entity, f func(Entity) error) error {
	key := Key(c, e)

	err := datastore.RunInTransaction(c, func(c appengine.Context) error {
		// reset slice fields (inside the transaction so it's retried)
		if x, ok := e.(NeedsIdempotentReset); ok {
			x.IdempotentReset()
		}

		// fetch most recent entity from datastore
		err := datastore.Get(c, key, e)
		if err == nil || IsErrFieldMismatch(err) {
			if x, ok := e.(HasGetHook); ok {
				x.HookAfterGet()
			}
		} else {
			return err
		}

		// perform the modifications
		err = f(e)
		if err != nil {
			return err
		}

		// write entity to datastore
		if x, ok := e.(HasPutHook); ok {
			x.HookBeforePut()
		}
		_, err = datastore.Put(c, key, e)
		return err
	}, nil)

	// did the transaction succeed?
	if err != nil {
		return err
	}

	// delete cache entry (See Note_1)
	err = ClearCache(c, e)
	if err != nil {
		return err
	}

	return nil
}

// Note_1
//
// Memcache operations are not transactional.  All combinations of commit
// and delete-from-cache leave some window of time during which the cache is
// stale.  The best we can do is minimize the size of this window.
//
// If we delete cache before our transaction, someone else might read a value
// and populate the cache just before our transaction commits. That leaves a
// permanent window of stale cache data. If we delete cache inside our
// transaction, we end have the same problem.
//
// By deleting cache right after we commit, there's a small window of time
// between commit and delete when someone might read and populate the cache with
// stale data.  Very soon afterwards, we delete the cache.  The window of stale
// date is on the order of 10 ms.  That's the best combination available to us.

func canBeCached(e Entity) bool {
	x, ok := e.(CanBeCached)
	return ok && x.CacheTtl() > 0
}

// StructProperties returns a slice of properties indicating how this struct
// would be saved to the datastore if one were to call datastore.SaveStruct() on
// it. The struct is not actually written to the datastore.  src must be a
// struct pointer.
func StructProperties(src interface{}) (datastore.PropertyList, error) {
	propCh := make(chan datastore.Property)
	errCh := make(chan error)

	go func() {
		errCh <- datastore.SaveStruct(src, propCh)
	}()

	props := make(datastore.PropertyList, 0)
	for prop := range propCh {
		props = append(props, prop)
	}

	return props, <-errCh
}
<commit_msg>Clarify how Modify and appengine.Context interact<commit_after>package aeds

import (
	"bytes"
	"encoding/gob"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/memcache"
)

// interface for structures that can be stored in App Engine's datastore
type Entity interface {
	Kind() string
	StringId() string
}

// HasGetHook is implemented by any Entity that wants to execute
// specific code after fetching the raw entity from datastore.
// This is often used to calculate derived fields.
type HasGetHook interface {
	HookAfterGet()
}

// HasPutHook is implemented by any Entity that wants to execute
// specific code before writing the raw entity to datastore.
// This is often used to calculate derived fields.
type HasPutHook interface {
	HookBeforePut()
}

// CanBeCached is implemented by any Entity that wants to
// have its values stored in memcache to improve read performance.
type CanBeCached interface {
	// CacheTtl indicates how long the entity should be cached in memcache.
	// Return zero to disable memcache.  If this method returns a non-zero
	// duration, the receiver should also implement the GobEncoder and
	// GobDecoder interfaces.
	CacheTtl() time.Duration
}

// NeedsIdempotentReset is implemented by any Entity that needs to reset its
// fields to zero when performing datastore read operations that are intended to
// be idempotent. For example, a datastore read into a slice property is not
// idempotent (it just appends property values to the slice).  In that case,
// this method might manually reset the slice length to 0 so that append leaves
// only the desired data.
type NeedsIdempotentReset interface {
	IdempotentReset()
}

// Key returns a datastore key for this entity.
func Key(c appengine.Context, e Entity) *datastore.Key {
	return datastore.NewKey(c, e.Kind(), e.StringId(), 0, nil)
}

// Put stores an entity in the datastore.
func Put(c appengine.Context, e Entity) (*datastore.Key, error) {
	if x, ok := e.(HasPutHook); ok {
		x.HookBeforePut()
	}

	// store entity in the datastore
	lookupKey := Key(c, e)
	key, err := datastore.Put(c, lookupKey, e)
	if err != nil {
		return nil, err
	}

	// delete from memcache?
	err = ClearCache(c, e)
	if err != nil {
		c.Errorf("aeds.Put ClearCache error: %s", err)
	}

	return key, nil
}

// ClearCache explicitly clears any memcache entries associated with this
// entity. One doesn't usually call this function directly.  Rather, it's called
// implicitly when other aeds functions know the cache should be cleared.
func ClearCache(c appengine.Context, e Entity) error {
	// nothing to do for uncacheable entities
	if !canBeCached(e) {
		return nil
	}

	err := memcache.Delete(c, Key(c, e).String())
	switch err {
	case nil:
	case memcache.ErrCacheMiss:
	default:
		return err
	}

	return nil
}

// Delete removes an entity from the datastore.
func Delete(c appengine.Context, e Entity) error {
	lookupKey := Key(c, e)

	// should the entity be removed from memcache too?
	err := ClearCache(c, e)
	if err != nil {
		return err
	}

	return datastore.Delete(c, lookupKey)
}

// FromId fetches an entity based on its ID.  The given entity
// should have enough data to calculate the entity's key.  On
// success, the entity is modified in place with all data from
// the datastore.
// Field mismatch errors are ignored.
func FromId(c appengine.Context, e Entity) (Entity, error) {
	lookupKey := Key(c, e)
	var ttl time.Duration
	if x, ok := e.(CanBeCached); ok {
		ttl = x.CacheTtl()
	}

	// should we look in memcache too?
	cacheMiss := false
	if ttl > 0 {
		item, err := memcache.Get(c, lookupKey.String())
		if err == nil {
			buf := bytes.NewBuffer(item.Value)
			err := gob.NewDecoder(buf).Decode(e)
			if x, ok := e.(HasGetHook); ok {
				x.HookAfterGet()
			}
			return e, err
		}
		if err == memcache.ErrCacheMiss {
			cacheMiss = true
		}
		// ignore any memcache errors
	}

	// look in the datastore
	err := datastore.Get(c, lookupKey, e)
	if err == nil || IsErrFieldMismatch(err) {
		if x, ok := e.(HasGetHook); ok {
			x.HookAfterGet()
		}

		// should we update memcache?
		if cacheMiss && ttl > 0 {
			if x, ok := e.(HasPutHook); ok {
				x.HookBeforePut()
			}

			// encode
			var value bytes.Buffer
			err := gob.NewEncoder(&value).Encode(e)
			if err != nil {
				return nil, err
			}

			// store
			item := &memcache.Item{
				Key:        lookupKey.String(),
				Value:      value.Bytes(),
				Expiration: ttl,
			}
			err = memcache.Set(c, item)
			_ = err // ignore memcache errors
		}

		return e, nil
	}
	return nil, err // unknown datastore error
}

// Modify atomically executes a read, modify, write operation on a single
// entity.  It should be used any time the results of a datastore read influence
// the contents of a datastore write.  Before executing f, the contents of e
// will be overwritten with the latest data available from the datastore.
//
// f should return an error value if something goes wrong with the modification.
// Modify returns that error value.
//
// As always, hooks defined by HookAfterGet() and HookBeforePut() are
// automatically executed at the appropriate time.  Be sure to define
// IdempotentReset() if your entity has any slice properties.
//
// You should not perform any datastore operations inside f.  By design, it
// doesn't have access to the transactional context used internally.  Other
// datastore changes will happen, even if the transaction fails to commit.
func Modify(c appengine.Context, e Entity, f func(Entity) error) error {
	key := Key(c, e)

	err := datastore.RunInTransaction(c, func(c appengine.Context) error {
		// reset slice fields (inside the transaction so it's retried)
		if x, ok := e.(NeedsIdempotentReset); ok {
			x.IdempotentReset()
		}

		// fetch most recent entity from datastore
		err := datastore.Get(c, key, e)
		if err == nil || IsErrFieldMismatch(err) {
			if x, ok := e.(HasGetHook); ok {
				x.HookAfterGet()
			}
		} else {
			return err
		}

		// perform the modifications
		err = f(e)
		if err != nil {
			return err
		}

		// write entity to datastore
		if x, ok := e.(HasPutHook); ok {
			x.HookBeforePut()
		}
		_, err = datastore.Put(c, key, e)
		return err
	}, nil)

	// did the transaction succeed?
	if err != nil {
		return err
	}

	// delete cache entry (See Note_1)
	err = ClearCache(c, e)
	if err != nil {
		return err
	}

	return nil
}

// Note_1
//
// Memcache operations are not transactional.  All combinations of commit
// and delete-from-cache leave some window of time during which the cache is
// stale.  The best we can do is minimize the size of this window.
//
// If we delete cache before our transaction, someone else might read a value
// and populate the cache just before our transaction commits. That leaves a
// permanent window of stale cache data. If we delete cache inside our
// transaction, we end have the same problem.
//
// By deleting cache right after we commit, there's a small window of time
// between commit and delete when someone might read and populate the cache with
// stale data.  Very soon afterwards, we delete the cache.  The window of stale
// date is on the order of 10 ms.  That's the best combination available to us.

func canBeCached(e Entity) bool {
	x, ok := e.(CanBeCached)
	return ok && x.CacheTtl() > 0
}

// StructProperties returns a slice of properties indicating how this struct
// would be saved to the datastore if one were to call datastore.SaveStruct() on
// it. The struct is not actually written to the datastore.  src must be a
// struct pointer.
func StructProperties(src interface{}) (datastore.PropertyList, error) {
	propCh := make(chan datastore.Property)
	errCh := make(chan error)

	go func() {
		errCh <- datastore.SaveStruct(src, propCh)
	}()

	props := make(datastore.PropertyList, 0)
	for prop := range propCh {
		props = append(props, prop)
	}

	return props, <-errCh
}
<|endoftext|>


<commit_before>// Copyright 2017 Xiaomi, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"math"
	"time"

	log "github.com/Sirupsen/logrus"

	pfc "github.com/niean/goperfcounter"
	cmodel "github.com/open-falcon/falcon-plus/common/model"
	cutils "github.com/open-falcon/falcon-plus/common/utils"

	"github.com/open-falcon/falcon-plus/modules/graph/g"
	"github.com/open-falcon/falcon-plus/modules/graph/index"
	"github.com/open-falcon/falcon-plus/modules/graph/proc"
	"github.com/open-falcon/falcon-plus/modules/graph/rrdtool"
	"github.com/open-falcon/falcon-plus/modules/graph/store"
)

type Graph int

func (this *Graph) GetRrd(key string, rrdfile *g.File) (err error) {
	var (
		md5    string
		dsType string
		step   int
	)
	if md5, dsType, step, err = g.SplitRrdCacheKey(key); err != nil {
		return err
	} else {
		rrdfile.Filename = g.RrdFileName(g.Config().RRD.Storage, md5, dsType, step)
	}

	items := store.GraphItems.PopAll(key)
	if len(items) > 0 {
		rrdtool.FlushFile(rrdfile.Filename, md5, items)
	}

	rrdfile.Body, err = rrdtool.ReadFile(rrdfile.Filename, md5)
	return
}

func (this *Graph) Ping(req cmodel.NullRpcRequest, resp *cmodel.SimpleRpcResponse) error {
	return nil
}

func (this *Graph) Send(items []*cmodel.GraphItem, resp *cmodel.SimpleRpcResponse) error {
	go handleItems(items)
	return nil
}

// 供外部调用、处理接收到的数据 的接口
func HandleItems(items []*cmodel.GraphItem) error {
	handleItems(items)
	return nil
}

func handleItems(items []*cmodel.GraphItem) {
	if items == nil {
		return
	}

	count := len(items)
	if count == 0 {
		return
	}

	cfg := g.Config()

	for i := 0; i < count; i++ {
		if items[i] == nil {
			continue
		}

		endpoint := items[i].Endpoint
		if !g.IsValidString(endpoint) {
			if cfg.Debug {
				log.Printf("invalid endpoint: %s", endpoint)
			}
			pfc.Meter("invalidEnpoint", 1)
			continue
		}

		counter := cutils.Counter(items[i].Metric, items[i].Tags)
		if !g.IsValidString(counter) {
			if cfg.Debug {
				log.Printf("invalid counter: %s/%s", endpoint, counter)
			}
			pfc.Meter("invalidCounter", 1)
			continue
		}

		dsType := items[i].DsType
		step := items[i].Step
		checksum := items[i].Checksum()
		key := g.FormRrdCacheKey(checksum, dsType, step)

		//statistics
		proc.GraphRpcRecvCnt.Incr()

		// To Graph
		first := store.GraphItems.First(key)
		if first != nil && items[i].Timestamp <= first.Timestamp {
			continue
		}
		store.GraphItems.PushFront(key, items[i], checksum, cfg)

		// To Index
		index.ReceiveItem(items[i], checksum)

		// To History
		store.AddItem(checksum, items[i])
	}
}

func (this *Graph) Query(param cmodel.GraphQueryParam, resp *cmodel.GraphQueryResponse) error {
	var (
		datas      []*cmodel.RRDData
		datas_size int
	)

	// statistics
	proc.GraphQueryCnt.Incr()

	cfg := g.Config()

	// form empty response
	resp.Values = []*cmodel.RRDData{}
	resp.Endpoint = param.Endpoint
	resp.Counter = param.Counter
	dsType, step, exists := index.GetTypeAndStep(param.Endpoint, param.Counter) // complete dsType and step
	if !exists {
		return nil
	}
	resp.DsType = dsType
	resp.Step = step

	start_ts := param.Start - param.Start%int64(step)
	end_ts := param.End - param.End%int64(step) + int64(step)
	if end_ts-start_ts-int64(step) < 1 {
		return nil
	}

	md5 := cutils.Md5(param.Endpoint + "/" + param.Counter)
	key := g.FormRrdCacheKey(md5, dsType, step)
	filename := g.RrdFileName(cfg.RRD.Storage, md5, dsType, step)

	// read cached items
	items, flag := store.GraphItems.FetchAll(key)
	items_size := len(items)

	if cfg.Migrate.Enabled && flag&g.GRAPH_F_MISS != 0 {
		node, _ := rrdtool.Consistent.Get(param.Endpoint + "/" + param.Counter)
		done := make(chan error, 1)
		res := &cmodel.GraphAccurateQueryResponse{}
		rrdtool.Net_task_ch[node] <- &rrdtool.Net_task_t{
			Method: rrdtool.NET_TASK_M_QUERY,
			Done:   done,
			Args:   param,
			Reply:  res,
		}
		<-done
		// fetch data from remote
		datas = res.Values
		datas_size = len(datas)
	} else {
		// read data from rrd file
		// 从RRD中获取数据不包含起始时间点
		// 例: start_ts=1484651400,step=60,则第一个数据时间为1484651460)
		datas, _ = rrdtool.Fetch(filename, md5, param.ConsolFun, start_ts-int64(step), end_ts, step)
		datas_size = len(datas)
	}

	nowTs := time.Now().Unix()
	lastUpTs := nowTs - nowTs%int64(step)
	rra1StartTs := lastUpTs - int64(rrdtool.RRA1PointCnt*step)

	// consolidated, do not merge
	if start_ts < rra1StartTs {
		resp.Values = datas
		goto _RETURN_OK
	}

	// no cached items, do not merge
	if items_size < 1 {
		resp.Values = datas
		goto _RETURN_OK
	}

	// merge
	{
		// fmt cached items
		var val cmodel.JsonFloat
		cache := make([]*cmodel.RRDData, 0)

		ts := items[0].Timestamp
		itemEndTs := items[items_size-1].Timestamp
		itemIdx := 0
		if dsType == g.DERIVE || dsType == g.COUNTER {
			for ts < itemEndTs {
				if itemIdx < items_size-1 && ts == items[itemIdx].Timestamp {
					if ts == items[itemIdx+1].Timestamp-int64(step) {
						val = cmodel.JsonFloat(items[itemIdx+1].Value-items[itemIdx].Value) / cmodel.JsonFloat(step)
					} else {
						val = cmodel.JsonFloat(math.NaN())
					}
					if val < 0 {
						val = cmodel.JsonFloat(math.NaN())
					}
					itemIdx++
				} else {
					// missing
					val = cmodel.JsonFloat(math.NaN())
				}

				if ts >= start_ts && ts <= end_ts {
					cache = append(cache, &cmodel.RRDData{Timestamp: ts, Value: val})
				}
				ts = ts + int64(step)
			}
		} else if dsType == g.GAUGE {
			for ts <= itemEndTs {
				if itemIdx < items_size && ts == items[itemIdx].Timestamp {
					val = cmodel.JsonFloat(items[itemIdx].Value)
					itemIdx++
				} else {
					// missing
					val = cmodel.JsonFloat(math.NaN())
				}

				if ts >= start_ts && ts <= end_ts {
					cache = append(cache, &cmodel.RRDData{Timestamp: ts, Value: val})
				}
				ts = ts + int64(step)
			}
		}
		cache_size := len(cache)

		// do merging
		merged := make([]*cmodel.RRDData, 0)
		if datas_size > 0 {
			for _, val := range datas {
				if val.Timestamp >= start_ts && val.Timestamp <= end_ts {
					merged = append(merged, val) //rrdtool返回的数据,时间戳是连续的、不会有跳点的情况
				}
			}
		}

		if cache_size > 0 {
			rrdDataSize := len(merged)
			lastTs := cache[0].Timestamp

			// find junction
			rrdDataIdx := 0
			for rrdDataIdx = rrdDataSize - 1; rrdDataIdx >= 0; rrdDataIdx-- {
				if merged[rrdDataIdx].Timestamp < cache[0].Timestamp {
					lastTs = merged[rrdDataIdx].Timestamp
					break
				}
			}

			// fix missing
			for ts := lastTs + int64(step); ts < cache[0].Timestamp; ts += int64(step) {
				merged = append(merged, &cmodel.RRDData{Timestamp: ts, Value: cmodel.JsonFloat(math.NaN())})
			}

			// merge cached items to result
			rrdDataIdx += 1
			for cacheIdx := 0; cacheIdx < cache_size; cacheIdx++ {
				if rrdDataIdx < rrdDataSize {
					if !math.IsNaN(float64(cache[cacheIdx].Value)) {
						merged[rrdDataIdx] = cache[cacheIdx]
					}
				} else {
					merged = append(merged, cache[cacheIdx])
				}
				rrdDataIdx++
			}
		}
		mergedSize := len(merged)

		// fmt result
		ret_size := int((end_ts - start_ts) / int64(step))
		if dsType == g.GAUGE {
			ret_size += 1
		}
		ret := make([]*cmodel.RRDData, ret_size, ret_size)
		mergedIdx := 0
		ts = start_ts
		for i := 0; i < ret_size; i++ {
			if mergedIdx < mergedSize && ts == merged[mergedIdx].Timestamp {
				ret[i] = merged[mergedIdx]
				mergedIdx++
			} else {
				ret[i] = &cmodel.RRDData{Timestamp: ts, Value: cmodel.JsonFloat(math.NaN())}
			}
			ts += int64(step)
		}
		resp.Values = ret
	}

_RETURN_OK:
	// statistics
	proc.GraphQueryItemCnt.IncrBy(int64(len(resp.Values)))
	return nil
}

//从内存索引、MySQL中删除counter，并从磁盘上删除对应rrd文件
func (this *Graph) Delete(params []*cmodel.GraphDeleteParam, resp *cmodel.GraphDeleteResp) error {
	resp = &cmodel.GraphDeleteResp{}
	for _, param := range params {
		err, tags := cutils.SplitTagsString(param.Tags)
		if err != nil {
			log.Error("invalid tags:", param.Tags, "error:", err)
			continue
		}

		var item *cmodel.GraphItem = &cmodel.GraphItem{
			Endpoint: param.Endpoint,
			Metric:   param.Metric,
			Tags:     tags,
			DsType:   param.DsType,
			Step:     param.Step,
		}
		index.RemoveItem(item)
	}

	return nil
}

func (this *Graph) Info(param cmodel.GraphInfoParam, resp *cmodel.GraphInfoResp) error {
	// statistics
	proc.GraphInfoCnt.Incr()

	dsType, step, exists := index.GetTypeAndStep(param.Endpoint, param.Counter)
	if !exists {
		return nil
	}

	md5 := cutils.Md5(param.Endpoint + "/" + param.Counter)
	filename := fmt.Sprintf("%s/%s/%s_%s_%d.rrd", g.Config().RRD.Storage, md5[0:2], md5, dsType, step)

	resp.ConsolFun = dsType
	resp.Step = step
	resp.Filename = filename

	return nil
}

func (this *Graph) Last(param cmodel.GraphLastParam, resp *cmodel.GraphLastResp) error {
	// statistics
	proc.GraphLastCnt.Incr()

	resp.Endpoint = param.Endpoint
	resp.Counter = param.Counter
	resp.Value = GetLast(param.Endpoint, param.Counter)

	return nil
}

func (this *Graph) LastRaw(param cmodel.GraphLastParam, resp *cmodel.GraphLastResp) error {
	// statistics
	proc.GraphLastRawCnt.Incr()

	resp.Endpoint = param.Endpoint
	resp.Counter = param.Counter
	resp.Value = GetLastRaw(param.Endpoint, param.Counter)

	return nil
}

// 非法值: ts=0,value无意义
func GetLast(endpoint, counter string) *cmodel.RRDData {
	dsType, step, exists := index.GetTypeAndStep(endpoint, counter)
	if !exists {
		return cmodel.NewRRDData(0, 0.0)
	}

	if dsType == g.GAUGE {
		return GetLastRaw(endpoint, counter)
	}

	if dsType == g.COUNTER || dsType == g.DERIVE {
		md5 := cutils.Md5(endpoint + "/" + counter)
		items := store.GetAllItems(md5)
		if len(items) < 2 {
			return cmodel.NewRRDData(0, 0.0)
		}

		f0 := items[0]
		f1 := items[1]
		delta_ts := f0.Timestamp - f1.Timestamp
		delta_v := f0.Value - f1.Value
		if delta_ts != int64(step) || delta_ts <= 0 {
			return cmodel.NewRRDData(0, 0.0)
		}
		if delta_v < 0 {
			// when cnt restarted, new cnt value would be zero, so fix it here
			delta_v = 0
		}

		return cmodel.NewRRDData(f0.Timestamp, delta_v/float64(delta_ts))
	}

	return cmodel.NewRRDData(0, 0.0)
}

// 非法值: ts=0,value无意义
func GetLastRaw(endpoint, counter string) *cmodel.RRDData {
	md5 := cutils.Md5(endpoint + "/" + counter)
	item := store.GetLastItem(md5)
	return cmodel.NewRRDData(item.Timestamp, item.Value)
}
<commit_msg>优化缓存中COUNTER类型数据的计算<commit_after>// Copyright 2017 Xiaomi, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"math"
	"time"

	log "github.com/Sirupsen/logrus"

	pfc "github.com/niean/goperfcounter"
	cmodel "github.com/open-falcon/falcon-plus/common/model"
	cutils "github.com/open-falcon/falcon-plus/common/utils"

	"github.com/open-falcon/falcon-plus/modules/graph/g"
	"github.com/open-falcon/falcon-plus/modules/graph/index"
	"github.com/open-falcon/falcon-plus/modules/graph/proc"
	"github.com/open-falcon/falcon-plus/modules/graph/rrdtool"
	"github.com/open-falcon/falcon-plus/modules/graph/store"
)

type Graph int

func (this *Graph) GetRrd(key string, rrdfile *g.File) (err error) {
	var (
		md5    string
		dsType string
		step   int
	)
	if md5, dsType, step, err = g.SplitRrdCacheKey(key); err != nil {
		return err
	} else {
		rrdfile.Filename = g.RrdFileName(g.Config().RRD.Storage, md5, dsType, step)
	}

	items := store.GraphItems.PopAll(key)
	if len(items) > 0 {
		rrdtool.FlushFile(rrdfile.Filename, md5, items)
	}

	rrdfile.Body, err = rrdtool.ReadFile(rrdfile.Filename, md5)
	return
}

func (this *Graph) Ping(req cmodel.NullRpcRequest, resp *cmodel.SimpleRpcResponse) error {
	return nil
}

func (this *Graph) Send(items []*cmodel.GraphItem, resp *cmodel.SimpleRpcResponse) error {
	go handleItems(items)
	return nil
}

// 供外部调用、处理接收到的数据 的接口
func HandleItems(items []*cmodel.GraphItem) error {
	handleItems(items)
	return nil
}

func handleItems(items []*cmodel.GraphItem) {
	if items == nil {
		return
	}

	count := len(items)
	if count == 0 {
		return
	}

	cfg := g.Config()

	for i := 0; i < count; i++ {
		if items[i] == nil {
			continue
		}

		endpoint := items[i].Endpoint
		if !g.IsValidString(endpoint) {
			if cfg.Debug {
				log.Printf("invalid endpoint: %s", endpoint)
			}
			pfc.Meter("invalidEnpoint", 1)
			continue
		}

		counter := cutils.Counter(items[i].Metric, items[i].Tags)
		if !g.IsValidString(counter) {
			if cfg.Debug {
				log.Printf("invalid counter: %s/%s", endpoint, counter)
			}
			pfc.Meter("invalidCounter", 1)
			continue
		}

		dsType := items[i].DsType
		step := items[i].Step
		checksum := items[i].Checksum()
		key := g.FormRrdCacheKey(checksum, dsType, step)

		//statistics
		proc.GraphRpcRecvCnt.Incr()

		// To Graph
		first := store.GraphItems.First(key)
		if first != nil && items[i].Timestamp <= first.Timestamp {
			continue
		}
		store.GraphItems.PushFront(key, items[i], checksum, cfg)

		// To Index
		index.ReceiveItem(items[i], checksum)

		// To History
		store.AddItem(checksum, items[i])
	}
}

func (this *Graph) Query(param cmodel.GraphQueryParam, resp *cmodel.GraphQueryResponse) error {
	var (
		datas      []*cmodel.RRDData
		datas_size int
	)

	// statistics
	proc.GraphQueryCnt.Incr()

	cfg := g.Config()

	// form empty response
	resp.Values = []*cmodel.RRDData{}
	resp.Endpoint = param.Endpoint
	resp.Counter = param.Counter
	dsType, step, exists := index.GetTypeAndStep(param.Endpoint, param.Counter) // complete dsType and step
	if !exists {
		return nil
	}
	resp.DsType = dsType
	resp.Step = step

	start_ts := param.Start - param.Start%int64(step)
	end_ts := param.End - param.End%int64(step) + int64(step)
	if end_ts-start_ts-int64(step) < 1 {
		return nil
	}

	md5 := cutils.Md5(param.Endpoint + "/" + param.Counter)
	key := g.FormRrdCacheKey(md5, dsType, step)
	filename := g.RrdFileName(cfg.RRD.Storage, md5, dsType, step)

	// read cached items
	items, flag := store.GraphItems.FetchAll(key)
	items_size := len(items)

	if cfg.Migrate.Enabled && flag&g.GRAPH_F_MISS != 0 {
		node, _ := rrdtool.Consistent.Get(param.Endpoint + "/" + param.Counter)
		done := make(chan error, 1)
		res := &cmodel.GraphAccurateQueryResponse{}
		rrdtool.Net_task_ch[node] <- &rrdtool.Net_task_t{
			Method: rrdtool.NET_TASK_M_QUERY,
			Done:   done,
			Args:   param,
			Reply:  res,
		}
		<-done
		// fetch data from remote
		datas = res.Values
		datas_size = len(datas)
	} else {
		// read data from rrd file
		// 从RRD中获取数据不包含起始时间点
		// 例: start_ts=1484651400,step=60,则第一个数据时间为1484651460)
		datas, _ = rrdtool.Fetch(filename, md5, param.ConsolFun, start_ts-int64(step), end_ts, step)
		datas_size = len(datas)
	}

	nowTs := time.Now().Unix()
	lastUpTs := nowTs - nowTs%int64(step)
	rra1StartTs := lastUpTs - int64(rrdtool.RRA1PointCnt*step)

	// consolidated, do not merge
	if start_ts < rra1StartTs {
		resp.Values = datas
		goto _RETURN_OK
	}

	// no cached items, do not merge
	if items_size < 1 {
		resp.Values = datas
		goto _RETURN_OK
	}

	// merge
	{
		// fmt cached items
		var val cmodel.JsonFloat
		cache := make([]*cmodel.RRDData, 0)

		ts := items[0].Timestamp
		itemEndTs := items[items_size-1].Timestamp
		itemIdx := 0
		if dsType == g.DERIVE || dsType == g.COUNTER {
			for ts < itemEndTs {
				if itemIdx < items_size-1 && ts == items[itemIdx].Timestamp {
					if ts == items[itemIdx+1].Timestamp-int64(step) && items[itemIdx+1].Value >= items[itemIdx].Value{
						val = cmodel.JsonFloat(items[itemIdx+1].Value-items[itemIdx].Value) / cmodel.JsonFloat(step)
					} else {
						val = cmodel.JsonFloat(math.NaN())
					}
					itemIdx++
				} else {
					// missing
					val = cmodel.JsonFloat(math.NaN())
				}

				if ts >= start_ts && ts <= end_ts {
					cache = append(cache, &cmodel.RRDData{Timestamp: ts, Value: val})
				}
				ts = ts + int64(step)
			}
		} else if dsType == g.GAUGE {
			for ts <= itemEndTs {
				if itemIdx < items_size && ts == items[itemIdx].Timestamp {
					val = cmodel.JsonFloat(items[itemIdx].Value)
					itemIdx++
				} else {
					// missing
					val = cmodel.JsonFloat(math.NaN())
				}

				if ts >= start_ts && ts <= end_ts {
					cache = append(cache, &cmodel.RRDData{Timestamp: ts, Value: val})
				}
				ts = ts + int64(step)
			}
		}
		cache_size := len(cache)

		// do merging
		merged := make([]*cmodel.RRDData, 0)
		if datas_size > 0 {
			for _, val := range datas {
				if val.Timestamp >= start_ts && val.Timestamp <= end_ts {
					merged = append(merged, val) //rrdtool返回的数据,时间戳是连续的、不会有跳点的情况
				}
			}
		}

		if cache_size > 0 {
			rrdDataSize := len(merged)
			lastTs := cache[0].Timestamp

			// find junction
			rrdDataIdx := 0
			for rrdDataIdx = rrdDataSize - 1; rrdDataIdx >= 0; rrdDataIdx-- {
				if merged[rrdDataIdx].Timestamp < cache[0].Timestamp {
					lastTs = merged[rrdDataIdx].Timestamp
					break
				}
			}

			// fix missing
			for ts := lastTs + int64(step); ts < cache[0].Timestamp; ts += int64(step) {
				merged = append(merged, &cmodel.RRDData{Timestamp: ts, Value: cmodel.JsonFloat(math.NaN())})
			}

			// merge cached items to result
			rrdDataIdx += 1
			for cacheIdx := 0; cacheIdx < cache_size; cacheIdx++ {
				if rrdDataIdx < rrdDataSize {
					if !math.IsNaN(float64(cache[cacheIdx].Value)) {
						merged[rrdDataIdx] = cache[cacheIdx]
					}
				} else {
					merged = append(merged, cache[cacheIdx])
				}
				rrdDataIdx++
			}
		}
		mergedSize := len(merged)

		// fmt result
		ret_size := int((end_ts - start_ts) / int64(step))
		if dsType == g.GAUGE {
			ret_size += 1
		}
		ret := make([]*cmodel.RRDData, ret_size, ret_size)
		mergedIdx := 0
		ts = start_ts
		for i := 0; i < ret_size; i++ {
			if mergedIdx < mergedSize && ts == merged[mergedIdx].Timestamp {
				ret[i] = merged[mergedIdx]
				mergedIdx++
			} else {
				ret[i] = &cmodel.RRDData{Timestamp: ts, Value: cmodel.JsonFloat(math.NaN())}
			}
			ts += int64(step)
		}
		resp.Values = ret
	}

_RETURN_OK:
	// statistics
	proc.GraphQueryItemCnt.IncrBy(int64(len(resp.Values)))
	return nil
}

//从内存索引、MySQL中删除counter，并从磁盘上删除对应rrd文件
func (this *Graph) Delete(params []*cmodel.GraphDeleteParam, resp *cmodel.GraphDeleteResp) error {
	resp = &cmodel.GraphDeleteResp{}
	for _, param := range params {
		err, tags := cutils.SplitTagsString(param.Tags)
		if err != nil {
			log.Error("invalid tags:", param.Tags, "error:", err)
			continue
		}

		var item *cmodel.GraphItem = &cmodel.GraphItem{
			Endpoint: param.Endpoint,
			Metric:   param.Metric,
			Tags:     tags,
			DsType:   param.DsType,
			Step:     param.Step,
		}
		index.RemoveItem(item)
	}

	return nil
}

func (this *Graph) Info(param cmodel.GraphInfoParam, resp *cmodel.GraphInfoResp) error {
	// statistics
	proc.GraphInfoCnt.Incr()

	dsType, step, exists := index.GetTypeAndStep(param.Endpoint, param.Counter)
	if !exists {
		return nil
	}

	md5 := cutils.Md5(param.Endpoint + "/" + param.Counter)
	filename := fmt.Sprintf("%s/%s/%s_%s_%d.rrd", g.Config().RRD.Storage, md5[0:2], md5, dsType, step)

	resp.ConsolFun = dsType
	resp.Step = step
	resp.Filename = filename

	return nil
}

func (this *Graph) Last(param cmodel.GraphLastParam, resp *cmodel.GraphLastResp) error {
	// statistics
	proc.GraphLastCnt.Incr()

	resp.Endpoint = param.Endpoint
	resp.Counter = param.Counter
	resp.Value = GetLast(param.Endpoint, param.Counter)

	return nil
}

func (this *Graph) LastRaw(param cmodel.GraphLastParam, resp *cmodel.GraphLastResp) error {
	// statistics
	proc.GraphLastRawCnt.Incr()

	resp.Endpoint = param.Endpoint
	resp.Counter = param.Counter
	resp.Value = GetLastRaw(param.Endpoint, param.Counter)

	return nil
}

// 非法值: ts=0,value无意义
func GetLast(endpoint, counter string) *cmodel.RRDData {
	dsType, step, exists := index.GetTypeAndStep(endpoint, counter)
	if !exists {
		return cmodel.NewRRDData(0, 0.0)
	}

	if dsType == g.GAUGE {
		return GetLastRaw(endpoint, counter)
	}

	if dsType == g.COUNTER || dsType == g.DERIVE {
		md5 := cutils.Md5(endpoint + "/" + counter)
		items := store.GetAllItems(md5)
		if len(items) < 2 {
			return cmodel.NewRRDData(0, 0.0)
		}

		f0 := items[0]
		f1 := items[1]
		delta_ts := f0.Timestamp - f1.Timestamp
		delta_v := f0.Value - f1.Value
		if delta_ts != int64(step) || delta_ts <= 0 {
			return cmodel.NewRRDData(0, 0.0)
		}
		if delta_v < 0 {
			// when cnt restarted, new cnt value would be zero, so fix it here
			delta_v = 0
		}

		return cmodel.NewRRDData(f0.Timestamp, delta_v/float64(delta_ts))
	}

	return cmodel.NewRRDData(0, 0.0)
}

// 非法值: ts=0,value无意义
func GetLastRaw(endpoint, counter string) *cmodel.RRDData {
	md5 := cutils.Md5(endpoint + "/" + counter)
	item := store.GetLastItem(md5)
	return cmodel.NewRRDData(item.Timestamp, item.Value)
}
<|endoftext|>