<commit_before>package state

import (
	"launchpad.net/goyaml"
	"launchpad.net/juju-core/juju/state/watcher"
	"launchpad.net/tomb"
)

// ConfigWatcher observes changes to any configuration node.
type ConfigWatcher struct {
	st         *State
	path       string
	tomb       tomb.Tomb
	watcher    *watcher.ContentWatcher
	changeChan chan *ConfigNode
}

// newConfigWatcher creates and starts a new config watcher for
// the given path.
func newConfigWatcher(st *State, path string) *ConfigWatcher {
	w := &ConfigWatcher{
		st:         st,
		path:       path,
		changeChan: make(chan *ConfigNode),
		watcher:    watcher.NewContentWatcher(st.zk, path),
	}
	go w.loop()
	return w
}

// Changes returns a channel that will receive the new
// *ConfigNode when a change is detected. Note that multiple
// changes may be observed as a single event in the channel.
// The first event on the channel holds the initial state
// as returned by Service.Config.
func (w *ConfigWatcher) Changes() <-chan *ConfigNode {
	return w.changeChan
}

// Stop stops the watch and returns any error encountered
// while watching. This method should always be called
// before discarding the watcher.
func (w *ConfigWatcher) Stop() error {
	w.tomb.Kill(nil)
	if err := w.watcher.Stop(); err != nil {
		w.tomb.Wait()
		return err
	}
	return w.tomb.Wait()
}

// loop is the backend for watching the configuration node.
func (w *ConfigWatcher) loop() {
	defer w.tomb.Done()
	defer close(w.changeChan)

	for {
		select {
		case <-w.tomb.Dying():
			return
		case change, ok := <-w.watcher.Changes():
			if !ok {
				return
			}
			// A non-existent node is treated as an empty node.
			configNode, err := parseConfigNode(w.st.zk, w.path, change.Content)
			if err != nil {
				w.tomb.Kill(err)
				return
			}
			select {
			case <-w.watcher.Dying():
				return
			case <-w.tomb.Dying():
				return
			case w.changeChan <- configNode:
			}
		}
	}
}

// NeedsUpgradeWatcher observes changes to a unit's upgrade flag.
type NeedsUpgradeWatcher struct {
	st         *State
	path       string
	tomb       tomb.Tomb
	watcher    *watcher.ContentWatcher
	changeChan chan NeedsUpgrade
}

// newNeedsUpgradeWatcher creates and starts a new resolved flag node
// watcher for the given path.
func newNeedsUpgradeWatcher(st *State, path string) *NeedsUpgradeWatcher {
	w := &NeedsUpgradeWatcher{
		st:         st,
		path:       path,
		changeChan: make(chan NeedsUpgrade),
		watcher:    watcher.NewContentWatcher(st.zk, path),
	}
	go w.loop()
	return w
}

// Changes returns a channel that will receive notifications
// about upgrades for the unit. Note that multiple changes
// may be observed as a single event in the channel.
// The first event on the channel holds the initial
// state as returned by Unit.NeedsUpgrade.
func (w *NeedsUpgradeWatcher) Changes() <-chan NeedsUpgrade {
	return w.changeChan
}

// Stop stops the watch and returns any error encountered
// while watching. This method should always be called
// before discarding the watcher.
func (w *NeedsUpgradeWatcher) Stop() error {
	w.tomb.Kill(nil)
	if err := w.watcher.Stop(); err != nil {
		w.tomb.Wait()
		return err
	}
	return w.tomb.Wait()
}

// loop is the backend for watching the resolved flag node.
func (w *NeedsUpgradeWatcher) loop() {
	defer w.tomb.Done()
	defer close(w.changeChan)

	for {
		select {
		case <-w.tomb.Dying():
			return
		case change, ok := <-w.watcher.Changes():
			if !ok {
				return
			}
			var needsUpgrade NeedsUpgrade
			if change.Exists {
				needsUpgrade.Upgrade = true
				var setting needsUpgradeNode
				if err := goyaml.Unmarshal([]byte(change.Content), &setting); err != nil {
					w.tomb.Kill(err)
					return
				}
				needsUpgrade.Force = setting.Force
			}
			select {
			case <-w.watcher.Dying():
				return
			case <-w.tomb.Dying():
				return
			case w.changeChan <- needsUpgrade:
			}
		}
	}
}

// ResolvedWatcher observes changes to a unit's resolved
// mode. See SetResolved for details.
type ResolvedWatcher struct {
	st         *State
	path       string
	tomb       tomb.Tomb
	watcher    *watcher.ContentWatcher
	changeChan chan ResolvedMode
}

// newResolvedWatcher returns a new ResolvedWatcher watching path.
func newResolvedWatcher(st *State, path string) *ResolvedWatcher {
	w := &ResolvedWatcher{
		st:         st,
		path:       path,
		changeChan: make(chan ResolvedMode),
		watcher:    watcher.NewContentWatcher(st.zk, path),
	}
	go w.loop()
	return w
}

// Changes returns a channel that will receive the new
// resolved mode when a change is detected. Note that multiple
// changes may be observed as a single event in the channel.
// The first event on the channel holds the initial
// state as returned by Unit.Resolved.
func (w *ResolvedWatcher) Changes() <-chan ResolvedMode {
	return w.changeChan
}

// Stop stops the watch and returns any error encountered
// while watching. This method should always be called
// before discarding the watcher.
func (w *ResolvedWatcher) Stop() error {
	w.tomb.Kill(nil)
	if err := w.watcher.Stop(); err != nil {
		w.tomb.Wait()
		return err
	}
	return w.tomb.Wait()
}

// loop is the backend for watching the resolved flag node.
func (w *ResolvedWatcher) loop() {
	defer w.tomb.Done()
	defer close(w.changeChan)

	for {
		select {
		case <-w.tomb.Dying():
			return
		case change, ok := <-w.watcher.Changes():
			if !ok {
				return
			}
			mode := ResolvedNone
			if change.Exists {
				var err error
				mode, err = parseResolvedMode(change.Content)
				if err != nil {
					w.tomb.Kill(err)
					return
				}
			}
			select {
			case <-w.watcher.Dying():
				return
			case <-w.tomb.Dying():
				return
			case w.changeChan <- mode:
			}
		}
	}
}

// PortsWatcher observes changes to a unit's open ports.
// See OpenPort for details.
type PortsWatcher struct {
	st         *State
	path       string
	tomb       tomb.Tomb
	watcher    *watcher.ContentWatcher
	changeChan chan []Port
}

// newPortsWatcher creates and starts a new ports node
// watcher for the given path.
func newPortsWatcher(st *State, path string) *PortsWatcher {
	w := &PortsWatcher{
		st:         st,
		path:       path,
		changeChan: make(chan []Port),
		watcher:    watcher.NewContentWatcher(st.zk, path),
	}
	go w.loop()
	return w
}

// Changes returns a channel that will receive the actual
// open ports when a change is detected. Note that multiple
// changes may be observed as a single event in the channel.
// The first event on the channel holds the initial
// state as returned by Unit.OpenPorts.
func (w *PortsWatcher) Changes() <-chan []Port {
	return w.changeChan
}

// Stop stops the watch and returns any error encountered
// while watching. This method should always be called
// before discarding the watcher.
func (w *PortsWatcher) Stop() error {
	w.tomb.Kill(nil)
	if err := w.watcher.Stop(); err != nil {
		w.tomb.Wait()
		return err
	}
	return w.tomb.Wait()
}

// loop is the backend for watching the ports node.
func (w *PortsWatcher) loop() {
	defer w.tomb.Done()
	defer close(w.changeChan)

	for {
		select {
		case <-w.tomb.Dying():
			return
		case change, ok := <-w.watcher.Changes():
			if !ok {
				return
			}
			var ports openPortsNode
			if err := goyaml.Unmarshal([]byte(change.Content), &ports); err != nil {
				w.tomb.Kill(err)
				return
			}
			select {
			case <-w.watcher.Dying():
				return
			case <-w.tomb.Dying():
				return
			case w.changeChan <- ports.Open:
			}
		}
	}
}

// MachinesWatcher notifies about machines being added or removed
// from the environment.
type MachinesWatcher struct {
	st               *State
	path             string
	tomb             tomb.Tomb
	changeChan       chan *MachinesChange
	watcher          *watcher.ContentWatcher
	knownMachineKeys []string
}

// newMachinesWatcher creates and starts a new machine watcher.
func newMachinesWatcher(st *State) *MachinesWatcher {
	// start with an empty topology
	topology, _ := parseTopology("")
	w := &MachinesWatcher{
		st:               st,
		path:             zkTopologyPath,
		changeChan:       make(chan *MachinesChange),
		watcher:          watcher.NewContentWatcher(st.zk, zkTopologyPath),
		knownMachineKeys: topology.MachineKeys(),
	}
	go w.loop()
	return w
}

// Changes returns a channel that will receive the actual
// watcher.ChildrenChanges. Note that multiple changes may
// be observed as a single event in the channel.
// The Added field in the first event on the channel holds the initial
// state as returned by State.AllMachines.
func (w *MachinesWatcher) Changes() <-chan *MachinesChange {
	return w.changeChan
}

// Stop stops the watch and returns any error encountered
// while watching. This method should always be called
// before discarding the watcher.
func (w *MachinesWatcher) Stop() error {
	w.tomb.Kill(nil)
	if err := w.watcher.Stop(); err != nil {
		w.tomb.Wait()
		return err
	}
	return w.tomb.Wait()
}

// loop is the backend for watching the ports node.
func (w *MachinesWatcher) loop() {
	defer w.tomb.Done()
	defer close(w.changeChan)
	for {
		select {
		case <-w.tomb.Dying():
			return
		case change, ok := <-w.watcher.Changes():
			if !ok {
				return
			}
			topology, err := parseTopology(change.Content)
			if err != nil {
				w.tomb.Kill(err)
				return
			}
			currentMachineKeys := topology.MachineKeys()
			added, deleted := diff(currentMachineKeys, w.knownMachineKeys), diff(w.knownMachineKeys, currentMachineKeys)
			w.knownMachineKeys = currentMachineKeys
			if len(added) == 0 && len(deleted) == 0 {
				// nothing changed in zkMachinePath
				continue
			}
			// Why are we dealing with strings, not *Machines at this point ?
			// Because *Machine does not define equality, yet.
			mc := new(MachinesChange)
			for _, m := range added {
				mc.Added = append(mc.Added, &Machine{w.st, m})
			}
			for _, m := range deleted {
				mc.Deleted = append(mc.Deleted, &Machine{w.st, m})
			}
			select {
			case <-w.watcher.Dying():
				return
			case <-w.tomb.Dying():
				return
			case w.changeChan <- mc:
			}
		}
	}
}

// diff returns all the elements that exist in A but not B.
func diff(A, B []string) (missing []string) {
next:
	for _, a := range A {
		for _, b := range B {
			if a == b {
				continue next
			}
		}
		missing = append(missing, a)
	}
	return
}
<commit_msg>state watchers now treat closure of underlying watchers as errors<commit_after>package state

import (
	"launchpad.net/goyaml"
	"launchpad.net/juju-core/juju/state/watcher"
	"launchpad.net/tomb"
)

// ConfigWatcher observes changes to any configuration node.
type ConfigWatcher struct {
	st         *State
	path       string
	tomb       tomb.Tomb
	watcher    *watcher.ContentWatcher
	changeChan chan *ConfigNode
}

// newConfigWatcher creates and starts a new config watcher for
// the given path.
func newConfigWatcher(st *State, path string) *ConfigWatcher {
	w := &ConfigWatcher{
		st:         st,
		path:       path,
		changeChan: make(chan *ConfigNode),
		watcher:    watcher.NewContentWatcher(st.zk, path),
	}
	go w.loop()
	return w
}

// Changes returns a channel that will receive the new
// *ConfigNode when a change is detected. Note that multiple
// changes may be observed as a single event in the channel.
// The first event on the channel holds the initial state
// as returned by Service.Config.
func (w *ConfigWatcher) Changes() <-chan *ConfigNode {
	return w.changeChan
}

// Stop stops the watch and returns any error encountered
// while watching. This method should always be called
// before discarding the watcher.
func (w *ConfigWatcher) Stop() error {
	w.tomb.Kill(nil)
	if err := w.watcher.Stop(); err != nil {
		w.tomb.Wait()
		return err
	}
	return w.tomb.Wait()
}

// loop is the backend for watching the configuration node.
func (w *ConfigWatcher) loop() {
	defer w.tomb.Done()
	defer close(w.changeChan)

	for {
		select {
		case <-w.tomb.Dying():
			return
		case change, ok := <-w.watcher.Changes():
			if !ok {
				w.tomb.Killf("content change channel closed unexpectedly")
				return
			}
			// A non-existent node is treated as an empty node.
			configNode, err := parseConfigNode(w.st.zk, w.path, change.Content)
			if err != nil {
				w.tomb.Kill(err)
				return
			}
			select {
			case <-w.tomb.Dying():
				return
			case w.changeChan <- configNode:
			}
		}
	}
}

// NeedsUpgradeWatcher observes changes to a unit's upgrade flag.
type NeedsUpgradeWatcher struct {
	st         *State
	path       string
	tomb       tomb.Tomb
	watcher    *watcher.ContentWatcher
	changeChan chan NeedsUpgrade
}

// newNeedsUpgradeWatcher creates and starts a new resolved flag node
// watcher for the given path.
func newNeedsUpgradeWatcher(st *State, path string) *NeedsUpgradeWatcher {
	w := &NeedsUpgradeWatcher{
		st:         st,
		path:       path,
		changeChan: make(chan NeedsUpgrade),
		watcher:    watcher.NewContentWatcher(st.zk, path),
	}
	go w.loop()
	return w
}

// Changes returns a channel that will receive notifications
// about upgrades for the unit. Note that multiple changes
// may be observed as a single event in the channel.
// The first event on the channel holds the initial
// state as returned by Unit.NeedsUpgrade.
func (w *NeedsUpgradeWatcher) Changes() <-chan NeedsUpgrade {
	return w.changeChan
}

// Stop stops the watch and returns any error encountered
// while watching. This method should always be called
// before discarding the watcher.
func (w *NeedsUpgradeWatcher) Stop() error {
	w.tomb.Kill(nil)
	if err := w.watcher.Stop(); err != nil {
		w.tomb.Wait()
		return err
	}
	return w.tomb.Wait()
}

// loop is the backend for watching the resolved flag node.
func (w *NeedsUpgradeWatcher) loop() {
	defer w.tomb.Done()
	defer close(w.changeChan)

	for {
		select {
		case <-w.tomb.Dying():
			return
		case change, ok := <-w.watcher.Changes():
			if !ok {
				w.tomb.Killf("content change channel closed unexpectedly")
				return
			}
			var needsUpgrade NeedsUpgrade
			if change.Exists {
				needsUpgrade.Upgrade = true
				var setting needsUpgradeNode
				if err := goyaml.Unmarshal([]byte(change.Content), &setting); err != nil {
					w.tomb.Kill(err)
					return
				}
				needsUpgrade.Force = setting.Force
			}
			select {
			case <-w.tomb.Dying():
				return
			case w.changeChan <- needsUpgrade:
			}
		}
	}
}

// ResolvedWatcher observes changes to a unit's resolved
// mode. See SetResolved for details.
type ResolvedWatcher struct {
	st         *State
	path       string
	tomb       tomb.Tomb
	watcher    *watcher.ContentWatcher
	changeChan chan ResolvedMode
}

// newResolvedWatcher returns a new ResolvedWatcher watching path.
func newResolvedWatcher(st *State, path string) *ResolvedWatcher {
	w := &ResolvedWatcher{
		st:         st,
		path:       path,
		changeChan: make(chan ResolvedMode),
		watcher:    watcher.NewContentWatcher(st.zk, path),
	}
	go w.loop()
	return w
}

// Changes returns a channel that will receive the new
// resolved mode when a change is detected. Note that multiple
// changes may be observed as a single event in the channel.
// The first event on the channel holds the initial
// state as returned by Unit.Resolved.
func (w *ResolvedWatcher) Changes() <-chan ResolvedMode {
	return w.changeChan
}

// Stop stops the watch and returns any error encountered
// while watching. This method should always be called
// before discarding the watcher.
func (w *ResolvedWatcher) Stop() error {
	w.tomb.Kill(nil)
	if err := w.watcher.Stop(); err != nil {
		w.tomb.Wait()
		return err
	}
	return w.tomb.Wait()
}

// loop is the backend for watching the resolved flag node.
func (w *ResolvedWatcher) loop() {
	defer w.tomb.Done()
	defer close(w.changeChan)

	for {
		select {
		case <-w.tomb.Dying():
			return
		case change, ok := <-w.watcher.Changes():
			if !ok {
				w.tomb.Killf("content change channel closed unexpectedly")
				return
			}
			mode := ResolvedNone
			if change.Exists {
				var err error
				mode, err = parseResolvedMode(change.Content)
				if err != nil {
					w.tomb.Kill(err)
					return
				}
			}
			select {
			case <-w.tomb.Dying():
				return
			case w.changeChan <- mode:
			}
		}
	}
}

// PortsWatcher observes changes to a unit's open ports.
// See OpenPort for details.
type PortsWatcher struct {
	st         *State
	path       string
	tomb       tomb.Tomb
	watcher    *watcher.ContentWatcher
	changeChan chan []Port
}

// newPortsWatcher creates and starts a new ports node
// watcher for the given path.
func newPortsWatcher(st *State, path string) *PortsWatcher {
	w := &PortsWatcher{
		st:         st,
		path:       path,
		changeChan: make(chan []Port),
		watcher:    watcher.NewContentWatcher(st.zk, path),
	}
	go w.loop()
	return w
}

// Changes returns a channel that will receive the actual
// open ports when a change is detected. Note that multiple
// changes may be observed as a single event in the channel.
// The first event on the channel holds the initial
// state as returned by Unit.OpenPorts.
func (w *PortsWatcher) Changes() <-chan []Port {
	return w.changeChan
}

// Stop stops the watch and returns any error encountered
// while watching. This method should always be called
// before discarding the watcher.
func (w *PortsWatcher) Stop() error {
	w.tomb.Kill(nil)
	if err := w.watcher.Stop(); err != nil {
		w.tomb.Wait()
		return err
	}
	return w.tomb.Wait()
}

// loop is the backend for watching the ports node.
func (w *PortsWatcher) loop() {
	defer w.tomb.Done()
	defer close(w.changeChan)

	for {
		select {
		case <-w.tomb.Dying():
			return
		case change, ok := <-w.watcher.Changes():
			if !ok {
				w.tomb.Killf("content change channel closed unexpectedly")
				return
			}
			var ports openPortsNode
			if err := goyaml.Unmarshal([]byte(change.Content), &ports); err != nil {
				w.tomb.Kill(err)
				return
			}
			select {
			case <-w.tomb.Dying():
				return
			case w.changeChan <- ports.Open:
			}
		}
	}
}

// MachinesWatcher notifies about machines being added or removed
// from the environment.
type MachinesWatcher struct {
	st               *State
	path             string
	tomb             tomb.Tomb
	changeChan       chan *MachinesChange
	watcher          *watcher.ContentWatcher
	knownMachineKeys []string
}

// newMachinesWatcher creates and starts a new machine watcher.
func newMachinesWatcher(st *State) *MachinesWatcher {
	// start with an empty topology
	topology, _ := parseTopology("")
	w := &MachinesWatcher{
		st:               st,
		path:             zkTopologyPath,
		changeChan:       make(chan *MachinesChange),
		watcher:          watcher.NewContentWatcher(st.zk, zkTopologyPath),
		knownMachineKeys: topology.MachineKeys(),
	}
	go w.loop()
	return w
}

// Changes returns a channel that will receive the actual
// watcher.ChildrenChanges. Note that multiple changes may
// be observed as a single event in the channel.
// The Added field in the first event on the channel holds the initial
// state as returned by State.AllMachines.
func (w *MachinesWatcher) Changes() <-chan *MachinesChange {
	return w.changeChan
}

// Stop stops the watch and returns any error encountered
// while watching. This method should always be called
// before discarding the watcher.
func (w *MachinesWatcher) Stop() error {
	w.tomb.Kill(nil)
	if err := w.watcher.Stop(); err != nil {
		w.tomb.Wait()
		return err
	}
	return w.tomb.Wait()
}

// loop is the backend for watching the ports node.
func (w *MachinesWatcher) loop() {
	defer w.tomb.Done()
	defer close(w.changeChan)
	for {
		select {
		case <-w.tomb.Dying():
			return
		case change, ok := <-w.watcher.Changes():
			if !ok {
				w.tomb.Killf("content change channel closed unexpectedly")
				return
			}
			topology, err := parseTopology(change.Content)
			if err != nil {
				w.tomb.Kill(err)
				return
			}
			currentMachineKeys := topology.MachineKeys()
			added, deleted := diff(currentMachineKeys, w.knownMachineKeys), diff(w.knownMachineKeys, currentMachineKeys)
			w.knownMachineKeys = currentMachineKeys
			if len(added) == 0 && len(deleted) == 0 {
				// The change was not relevant to this watcher.
				continue
			}
			// Why are we dealing with strings, not *Machines at this point ?
			// Because *Machine does not define equality, yet.
			mc := new(MachinesChange)
			for _, m := range added {
				mc.Added = append(mc.Added, &Machine{w.st, m})
			}
			for _, m := range deleted {
				mc.Deleted = append(mc.Deleted, &Machine{w.st, m})
			}
			select {
			case <-w.tomb.Dying():
				return
			case w.changeChan <- mc:
			}
		}
	}
}

// diff returns all the elements that exist in A but not B.
func diff(A, B []string) (missing []string) {
next:
	for _, a := range A {
		for _, b := range B {
			if a == b {
				continue next
			}
		}
		missing = append(missing, a)
	}
	return
}
<|endoftext|>


<commit_before>package vfs

import (
	"encoding/json"
	"path"
)

// FsckLogType is the type of a FsckLog
type FsckLogType int

const (
	// FileMissing used when a file data is missing from its index entry.
	FileMissing FsckLogType = iota
	// IndexMissing is used when the index entry is missing from a file data.
	IndexMissing
	// TypeMismatch is used when a document type does not match in the index and
	// underlying filesystem.
	TypeMismatch
)

// FsckLog is a struct for an inconsistency in the VFS
type FsckLog struct {
	Type        FsckLogType
	FileDoc     *FileDoc
	IsFile      bool
	DirDoc      *DirDoc
	Filename    string
	PruneAction string
	PruneError  error
}

// String returns a string describing the FsckLog
func (f *FsckLog) String() string {
	switch f.Type {
	case FileMissing:
		if f.IsFile {
			return "the file is present in the index but not on the filesystem"
		}
		return "the directory is present in the index but not on the filesystem"
	case TypeMismatch:
		if f.IsFile {
			return "it's a file in the index but a directory on the filesystem"
		}
		return "it's a directory in the index but a file on the filesystem"
	case IndexMissing:
		return "the document is present on the local filesystem but not in the index"
	}
	panic("bad FsckLog type")
}

// MarshalJSON implements the json.Marshaler interface
func (f *FsckLog) MarshalJSON() ([]byte, error) {
	v := map[string]string{
		"filename": f.Filename,
		"message":  f.String(),
	}
	if f.IsFile {
		v["file_id"] = f.FileDoc.ID()
	} else {
		v["file_id"] = f.DirDoc.ID()
	}
	if f.PruneAction != "" {
		v["prune_action"] = f.PruneAction
		v["prune_error"] = f.PruneError.Error()
	}
	return json.Marshal(v)
}

// FsckPrune tries to fix the given list on inconsistencies in the VFS
func FsckPrune(fs VFS, indexer Indexer, entry *FsckLog, dryrun bool) {
	switch entry.Type {
	case FileMissing:
		entry.PruneAction = "deleting entry from index"
		if dryrun {
			return
		}
		if entry.IsFile {
			if err := indexer.DeleteFileDoc(entry.FileDoc); err != nil {
				entry.PruneError = err
			}
		} else {
			if err := indexer.DeleteDirDoc(entry.DirDoc); err != nil {
				entry.PruneError = err
			}
		}
	case IndexMissing:
		if !entry.IsFile {
			return
		}
		fileDoc := entry.FileDoc
		var orphan bool
		parentDir, err := indexer.DirByID(fileDoc.DirID)
		if err != nil {
			orphan = true
		} else {
			fullpath := path.Join(parentDir.Fullpath, fileDoc.Name())
			if _, err := indexer.FileByPath(fullpath); err == nil {
				entry.PruneError = err
				return
			}
		}
		if orphan {
			entry.PruneAction = "creating index entry in orphan directory"
		} else {
			entry.PruneAction = "creating index entry in-place"
		}
		if dryrun {
			return
		}
		if orphan {
			orphanDir, err := Mkdir(fs, OrphansDirName, nil)
			if err != nil {
				entry.PruneError = err
				return
			}
			fileDoc.DirID = orphanDir.ID()
		}
		if err := indexer.CreateFileDoc(fileDoc); err != nil {
			entry.PruneError = err
		}
	}
}
<commit_msg>Fix error check<commit_after>package vfs

import (
	"encoding/json"
	"path"
)

// FsckLogType is the type of a FsckLog
type FsckLogType int

const (
	// FileMissing used when a file data is missing from its index entry.
	FileMissing FsckLogType = iota
	// IndexMissing is used when the index entry is missing from a file data.
	IndexMissing
	// TypeMismatch is used when a document type does not match in the index and
	// underlying filesystem.
	TypeMismatch
)

// FsckLog is a struct for an inconsistency in the VFS
type FsckLog struct {
	Type        FsckLogType
	FileDoc     *FileDoc
	IsFile      bool
	DirDoc      *DirDoc
	Filename    string
	PruneAction string
	PruneError  error
}

// String returns a string describing the FsckLog
func (f *FsckLog) String() string {
	switch f.Type {
	case FileMissing:
		if f.IsFile {
			return "the file is present in the index but not on the filesystem"
		}
		return "the directory is present in the index but not on the filesystem"
	case TypeMismatch:
		if f.IsFile {
			return "it's a file in the index but a directory on the filesystem"
		}
		return "it's a directory in the index but a file on the filesystem"
	case IndexMissing:
		return "the document is present on the local filesystem but not in the index"
	}
	panic("bad FsckLog type")
}

// MarshalJSON implements the json.Marshaler interface
func (f *FsckLog) MarshalJSON() ([]byte, error) {
	v := map[string]string{
		"filename": f.Filename,
		"message":  f.String(),
	}
	if f.IsFile {
		v["file_id"] = f.FileDoc.ID()
	} else {
		v["file_id"] = f.DirDoc.ID()
	}
	if f.PruneAction != "" {
		v["prune_action"] = f.PruneAction
		v["prune_error"] = f.PruneError.Error()
	}
	return json.Marshal(v)
}

// FsckPrune tries to fix the given list on inconsistencies in the VFS
func FsckPrune(fs VFS, indexer Indexer, entry *FsckLog, dryrun bool) {
	switch entry.Type {
	case FileMissing:
		entry.PruneAction = "deleting entry from index"
		if dryrun {
			return
		}
		if entry.IsFile {
			if err := indexer.DeleteFileDoc(entry.FileDoc); err != nil {
				entry.PruneError = err
			}
		} else {
			if err := indexer.DeleteDirDoc(entry.DirDoc); err != nil {
				entry.PruneError = err
			}
		}
	case IndexMissing:
		if !entry.IsFile {
			return
		}
		fileDoc := entry.FileDoc
		var orphan bool
		parentDir, err := indexer.DirByID(fileDoc.DirID)
		if err != nil {
			orphan = true
		} else {
			fullpath := path.Join(parentDir.Fullpath, fileDoc.Name())
			if _, err := indexer.FileByPath(fullpath); err != nil {
				entry.PruneError = err
				return
			}
		}
		if orphan {
			entry.PruneAction = "creating index entry in orphan directory"
		} else {
			entry.PruneAction = "creating index entry in-place"
		}
		if dryrun {
			return
		}
		if orphan {
			orphanDir, err := Mkdir(fs, OrphansDirName, nil)
			if err != nil {
				entry.PruneError = err
				return
			}
			fileDoc.DirID = orphanDir.ID()
		}
		if err := indexer.CreateFileDoc(fileDoc); err != nil {
			entry.PruneError = err
		}
	}
}
<|endoftext|>


<commit_before>package checkcloudwatchlogs

import (
	"fmt"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/jessevdk/go-flags"
	"github.com/pkg/errors"

	"github.com/mackerelio/checkers"
)

type logOpts struct {
	Region          string `long:"region" value-name:"REGION" description:"AWS Region"`
	AccessKeyID     string `long:"access-key-id" value-name:"ACCESS-KEY-ID" description:"AWS Access Key ID"`
	SecretAccessKey string `long:"secret-access-key" value-name:"SECRET-ACCESS-KEY" description:"AWS Secret Access Key"`
	LogGroupName    string `long:"log-group-name" value-name:"LOG-GROUP-NAME" description:"Log group name"`
}

// Do the plugin
func Do() {
	ckr := run(os.Args[1:])
	ckr.Name = "CloudWatch Logs"
	ckr.Exit()
}

type cloudwatchLogsPlugin struct {
	Service      *cloudwatchlogs.CloudWatchLogs
	LogGroupName string
}

func newCloudwatchLogsPlugin(args []string) (*cloudwatchLogsPlugin, error) {
	opts := &logOpts{}
	_, err := flags.ParseArgs(opts, args)
	if err != nil {
		return nil, err
	}
	service, err := createService(opts)
	if err != nil {
		return nil, err
	}
	return &cloudwatchLogsPlugin{
		Service:      service,
		LogGroupName: opts.LogGroupName,
	}, nil
}

func createService(opts *logOpts) (*cloudwatchlogs.CloudWatchLogs, error) {
	sess, err := session.NewSession()
	if err != nil {
		return nil, err
	}
	config := aws.NewConfig()
	if opts.AccessKeyID != "" && opts.SecretAccessKey != "" {
		config = config.WithCredentials(
			credentials.NewStaticCredentials(opts.AccessKeyID, opts.SecretAccessKey, ""),
		)
	}
	if opts.Region != "" {
		config = config.WithRegion(opts.Region)
	}
	return cloudwatchlogs.New(sess, config), nil
}

func (p *cloudwatchLogsPlugin) run() error {
	if p.LogGroupName == "" {
		return errors.New("specify log group name")
	}
	var nextToken *string
	for {
		startTime := time.Now().Add(-5 * time.Minute)
		output, err := p.Service.FilterLogEvents(&cloudwatchlogs.FilterLogEventsInput{
			StartTime:    aws.Int64(startTime.Unix() * 1000),
			LogGroupName: aws.String(p.LogGroupName),
			NextToken:    nextToken,
		})
		if err != nil {
			return err
		}
		fmt.Printf("%#v\n", err)
		fmt.Printf("%#v\n", output)
		if output.NextToken == nil {
			break
		}
		nextToken = output.NextToken
		time.Sleep(200 * time.Millisecond)
	}
	return nil
}

func run(args []string) *checkers.Checker {
	p, err := newCloudwatchLogsPlugin(args)
	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprint(err))
	}
	err = p.run()
	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprint(err))
	}
	return checkers.NewChecker(checkers.OK, "ok")
}
<commit_msg>use api interface<commit_after>package checkcloudwatchlogs

import (
	"fmt"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
	"github.com/jessevdk/go-flags"
	"github.com/pkg/errors"

	"github.com/mackerelio/checkers"
)

type logOpts struct {
	Region          string `long:"region" value-name:"REGION" description:"AWS Region"`
	AccessKeyID     string `long:"access-key-id" value-name:"ACCESS-KEY-ID" description:"AWS Access Key ID"`
	SecretAccessKey string `long:"secret-access-key" value-name:"SECRET-ACCESS-KEY" description:"AWS Secret Access Key"`
	LogGroupName    string `long:"log-group-name" value-name:"LOG-GROUP-NAME" description:"Log group name"`
}

// Do the plugin
func Do() {
	ckr := run(os.Args[1:])
	ckr.Name = "CloudWatch Logs"
	ckr.Exit()
}

type cloudwatchLogsPlugin struct {
	Service      cloudwatchlogsiface.CloudWatchLogsAPI
	LogGroupName string
}

func newCloudwatchLogsPlugin(args []string) (*cloudwatchLogsPlugin, error) {
	opts := &logOpts{}
	_, err := flags.ParseArgs(opts, args)
	if err != nil {
		return nil, err
	}
	service, err := createService(opts)
	if err != nil {
		return nil, err
	}
	return &cloudwatchLogsPlugin{
		Service:      service,
		LogGroupName: opts.LogGroupName,
	}, nil
}

func createService(opts *logOpts) (*cloudwatchlogs.CloudWatchLogs, error) {
	sess, err := session.NewSession()
	if err != nil {
		return nil, err
	}
	config := aws.NewConfig()
	if opts.AccessKeyID != "" && opts.SecretAccessKey != "" {
		config = config.WithCredentials(
			credentials.NewStaticCredentials(opts.AccessKeyID, opts.SecretAccessKey, ""),
		)
	}
	if opts.Region != "" {
		config = config.WithRegion(opts.Region)
	}
	return cloudwatchlogs.New(sess, config), nil
}

func (p *cloudwatchLogsPlugin) run() error {
	if p.LogGroupName == "" {
		return errors.New("specify log group name")
	}
	var nextToken *string
	for {
		startTime := time.Now().Add(-5 * time.Minute)
		output, err := p.Service.FilterLogEvents(&cloudwatchlogs.FilterLogEventsInput{
			StartTime:    aws.Int64(startTime.Unix() * 1000),
			LogGroupName: aws.String(p.LogGroupName),
			NextToken:    nextToken,
		})
		if err != nil {
			return err
		}
		fmt.Printf("%#v\n", err)
		fmt.Printf("%#v\n", output)
		if output.NextToken == nil {
			break
		}
		nextToken = output.NextToken
		time.Sleep(200 * time.Millisecond)
	}
	return nil
}

func run(args []string) *checkers.Checker {
	p, err := newCloudwatchLogsPlugin(args)
	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprint(err))
	}
	err = p.run()
	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprint(err))
	}
	return checkers.NewChecker(checkers.OK, "ok")
}
<|endoftext|>


<commit_before>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file contains the concurrency primitive functions.

package golisp

import (
	"fmt"
	"runtime"
	"strings"
	"sync/atomic"
	"time"
	"unsafe"
)

type Process struct {
	Env           *SymbolTableFrame
	Code          *Data
	Wake          chan bool
	Abort         chan bool
	Restart       chan bool
	ReturnValue   chan *Data
	Joined        int32
	ScheduleTimer *time.Timer
}

func RegisterConcurrencyPrimitives() {
	MakePrimitiveFunction("fork", "1", ForkImpl)
	MakePrimitiveFunction("proc-sleep", "2", ProcSleepImpl)
	MakePrimitiveFunction("wake", "1", WakeImpl)
	MakePrimitiveFunction("schedule", "2", ScheduleImpl)
	MakePrimitiveFunction("reset-timeout", "1", ResetTimeoutImpl)
	MakePrimitiveFunction("abandon", "1", AbandonImpl)
	MakePrimitiveFunction("join", "1", JoinImpl)
}

func ForkImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := Car(args)

	if !FunctionP(f) {
		err = ProcessError(fmt.Sprintf("fork expected a function, but received %v.", f), env)
		return
	}

	if FunctionValue(f).RequiredArgCount != 1 {
		err = ProcessError(fmt.Sprintf("fork expected a function with arity of 1, but it was %d.", FunctionValue(f).RequiredArgCount), env)
		return
	}

	proc := &Process{
		Env:         env,
		Code:        f,
		Wake:        make(chan bool, 1),
		Abort:       make(chan bool, 1),
		Restart:     make(chan bool, 1),
		ReturnValue: make(chan *Data, 1)}
	procObj := ObjectWithTypeAndValue("Process", unsafe.Pointer(proc))

	go func() {
		var returnValue *Data
		defer func() {
			proc.ReturnValue <- returnValue
		}()

		callWithPanicProtection(func() {
			var forkedErr error
			returnValue, forkedErr = FunctionValue(f).ApplyWithoutEval(InternalMakeList(procObj), env)
			if forkedErr != nil {
				fmt.Println(forkedErr)
			}
		}, "fork")
	}()

	return procObj, nil
}

func ProcSleepImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	procObj := Car(args)

	if !ObjectP(procObj) || ObjectType(procObj) != "Process" {
		err = ProcessError(fmt.Sprintf("proc-sleep expects a Process object expected but received %s.", ObjectType(procObj)), env)
		return
	}

	proc := (*Process)(ObjectValue(procObj))

	millis := Cadr(args)
	if !IntegerP(millis) {
		err = ProcessError(fmt.Sprintf("proc-sleep expected an integer as a delay, but received %v.", millis), env)
		return
	}

	woken := false
	select {
	case <-proc.Wake:
		woken = true
	case <-time.After(time.Duration(IntegerValue(millis)) * time.Millisecond):
	}

	return BooleanWithValue(woken), nil
}

func WakeImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	procObj := Car(args)

	if !ObjectP(procObj) || ObjectType(procObj) != "Process" {
		err = ProcessError(fmt.Sprintf("wake expects a Process object expected but received %s.", ObjectType(procObj)), env)
		return
	}

	proc := (*Process)(ObjectValue(procObj))
	proc.Wake <- true
	return StringWithValue("OK"), nil
}

func ScheduleImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	millis := Car(args)
	if !IntegerP(millis) {
		err = ProcessError(fmt.Sprintf("schedule expected an integer as a delay, but received %v.", millis), env)
		return
	}
	f := Cadr(args)

	if !FunctionP(f) {
		err = ProcessError(fmt.Sprintf("schedule expected a function, but received %v.", f), env)
		return
	}

	if FunctionValue(f).RequiredArgCount != 1 {
		err = ProcessError(fmt.Sprintf("schedule expected a function with arity of 1, but it was %d.", FunctionValue(f).RequiredArgCount), env)
		return
	}

	proc := &Process{
		Env:           env,
		Code:          f,
		Wake:          make(chan bool, 1),
		Abort:         make(chan bool, 1),
		Restart:       make(chan bool, 1),
		ReturnValue:   make(chan *Data, 1),
		ScheduleTimer: time.NewTimer(time.Duration(IntegerValue(millis)) * time.Millisecond)}
	procObj := ObjectWithTypeAndValue("Process", unsafe.Pointer(proc))

	aborted := false
	go func() {
		var returnValue *Data
		defer func() {
			proc.ReturnValue <- returnValue
		}()
		callWithPanicProtection(func() {
		Loop:
			for {
				select {
				case <-proc.Abort:
					aborted = true
					break Loop
				case <-proc.Restart:
					proc.ScheduleTimer.Reset(time.Duration(IntegerValue(millis)) * time.Millisecond)
				case <-proc.ScheduleTimer.C:
					var forkedErr error
					returnValue, forkedErr = FunctionValue(f).ApplyWithoutEval(InternalMakeList(procObj), env)
					if forkedErr != nil {
						fmt.Println(forkedErr)
					}
					break Loop
				}
			}
		}, "schedule")
	}()

	return procObj, nil

}

func AbandonImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	procObj := Car(args)

	if !ObjectP(procObj) || ObjectType(procObj) != "Process" {
		err = ProcessError(fmt.Sprintf("adandon expects a Process object expected but received %s.", ObjectType(procObj)), env)
		return
	}

	proc := (*Process)(ObjectValue(procObj))

	if proc.ScheduleTimer == nil {
		return nil, ProcessError("tried to adandon a Process that isn't scheduled", env)
	}

	proc.Abort <- true
	return StringWithValue("OK"), nil
}

func ResetTimeoutImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	procObj := Car(args)

	if !ObjectP(procObj) || ObjectType(procObj) != "Process" {
		err = ProcessError(fmt.Sprintf("restart expects a Process object expected but received %s.", ObjectType(procObj)), env)
		return
	}

	proc := (*Process)(ObjectValue(procObj))

	if proc.ScheduleTimer == nil {
		return nil, ProcessError("tried to reset a Process that isn't scheduled", env)
	}

	var str string
	select {
	case proc.Restart <- true:
		str = "OK"
	default:
		str = "task was already completed or abandoned"
	}
	return StringWithValue(str), nil
}

func JoinImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	procObj := Car(args)

	if !ObjectP(procObj) || ObjectType(procObj) != "Process" {
		err = ProcessError(fmt.Sprintf("join expects a Process object expected but received %s.", ObjectType(procObj)), env)
		return
	}
	proc := (*Process)(ObjectValue(procObj))

	if atomic.CompareAndSwapInt32(&proc.Joined, 0, 1) {
		return <-proc.ReturnValue, nil
	} else {
		return nil, ProcessError("tried to join on a task twice", env)
	}
}

func callWithPanicProtection(f func(), prefix string) {
	defer func() {
		if recovered := recover(); recovered != nil {
			stackBuf := make([]byte, 10000)
			stackBuf = stackBuf[:runtime.Stack(stackBuf, false)]
			stack := strings.Split(string(stackBuf), "\n")
			for i := 0; i < 7; i++ {
				fmt.Println(stack[i])
			}
		}
	}()

	f()
}
<commit_msg>allow passing arguments to forked tasks<commit_after>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file contains the concurrency primitive functions.

package golisp

import (
	"fmt"
	"runtime"
	"strings"
	"sync/atomic"
	"time"
	"unsafe"
)

type Process struct {
	Env           *SymbolTableFrame
	Code          *Data
	Wake          chan bool
	Abort         chan bool
	Restart       chan bool
	ReturnValue   chan *Data
	Joined        int32
	ScheduleTimer *time.Timer
}

func RegisterConcurrencyPrimitives() {
	MakePrimitiveFunction("fork", ">=1", ForkImpl)
	MakePrimitiveFunction("proc-sleep", "2", ProcSleepImpl)
	MakePrimitiveFunction("wake", "1", WakeImpl)
	MakePrimitiveFunction("schedule", ">=2", ScheduleImpl)
	MakePrimitiveFunction("reset-timeout", "1", ResetTimeoutImpl)
	MakePrimitiveFunction("abandon", "1", AbandonImpl)
	MakePrimitiveFunction("join", "1", JoinImpl)
}

func ForkImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	f := Car(args)

	if !FunctionP(f) {
		err = ProcessError(fmt.Sprintf("fork expected a function, but received %v.", f), env)
		return
	}

	argsCount := Length(Cdr(args)) + 1
	function := FunctionValue(f)

	if function.VarArgs {
		if argsCount < function.RequiredArgCount {
			return nil, ProcessError(fmt.Sprintf("fork expected a function with arity of at most %d, but it was %d.", argsCount, function.RequiredArgCount), env)
		}
	} else {
		if argsCount != function.RequiredArgCount {
			return nil, ProcessError(fmt.Sprintf("fork expected a function with arity of %d, but it was %d.", argsCount, function.RequiredArgCount), env)
		}
	}

	proc := &Process{
		Env:         env,
		Code:        f,
		Wake:        make(chan bool, 1),
		Abort:       make(chan bool, 1),
		Restart:     make(chan bool, 1),
		ReturnValue: make(chan *Data, 1)}
	procObj := ObjectWithTypeAndValue("Process", unsafe.Pointer(proc))

	go func() {
		var returnValue *Data
		defer func() {
			proc.ReturnValue <- returnValue
		}()

		callWithPanicProtection(func() {
			var forkedErr error
			returnValue, forkedErr = function.ApplyWithoutEval(Cons(procObj, Cdr(args)), env)
			if forkedErr != nil {
				fmt.Println(forkedErr)
			}
		}, "fork")
	}()

	return procObj, nil
}

func ProcSleepImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	procObj := Car(args)

	if !ObjectP(procObj) || ObjectType(procObj) != "Process" {
		err = ProcessError(fmt.Sprintf("proc-sleep expects a Process object expected but received %s.", ObjectType(procObj)), env)
		return
	}

	proc := (*Process)(ObjectValue(procObj))

	millis := Cadr(args)
	if !IntegerP(millis) {
		err = ProcessError(fmt.Sprintf("proc-sleep expected an integer as a delay, but received %v.", millis), env)
		return
	}

	woken := false
	select {
	case <-proc.Wake:
		woken = true
	case <-time.After(time.Duration(IntegerValue(millis)) * time.Millisecond):
	}

	return BooleanWithValue(woken), nil
}

func WakeImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	procObj := Car(args)

	if !ObjectP(procObj) || ObjectType(procObj) != "Process" {
		err = ProcessError(fmt.Sprintf("wake expects a Process object expected but received %s.", ObjectType(procObj)), env)
		return
	}

	proc := (*Process)(ObjectValue(procObj))
	proc.Wake <- true
	return StringWithValue("OK"), nil
}

func ScheduleImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	millis := Car(args)
	if !IntegerP(millis) {
		err = ProcessError(fmt.Sprintf("schedule expected an integer as a delay, but received %v.", millis), env)
		return
	}
	f := Cadr(args)

	if !FunctionP(f) {
		err = ProcessError(fmt.Sprintf("schedule expected a function, but received %v.", f), env)
		return
	}

	argsCount := Length(Cddr(args)) + 1
	function := FunctionValue(f)

	if function.VarArgs {
		if argsCount < function.RequiredArgCount {
			return nil, ProcessError(fmt.Sprintf("schedule expected a function with arity of at most %d, but it was %d.", argsCount, function.RequiredArgCount), env)
		}
	} else {
		if argsCount != function.RequiredArgCount {
			return nil, ProcessError(fmt.Sprintf("schedule expected a function with arity of %d, but it was %d.", argsCount, function.RequiredArgCount), env)
		}
	}

	proc := &Process{
		Env:           env,
		Code:          f,
		Wake:          make(chan bool, 1),
		Abort:         make(chan bool, 1),
		Restart:       make(chan bool, 1),
		ReturnValue:   make(chan *Data, 1),
		ScheduleTimer: time.NewTimer(time.Duration(IntegerValue(millis)) * time.Millisecond)}
	procObj := ObjectWithTypeAndValue("Process", unsafe.Pointer(proc))

	aborted := false
	go func() {
		var returnValue *Data
		defer func() {
			proc.ReturnValue <- returnValue
		}()
		callWithPanicProtection(func() {
		Loop:
			for {
				select {
				case <-proc.Abort:
					aborted = true
					break Loop
				case <-proc.Restart:
					proc.ScheduleTimer.Reset(time.Duration(IntegerValue(millis)) * time.Millisecond)
				case <-proc.ScheduleTimer.C:
					var forkedErr error
					returnValue, forkedErr = function.ApplyWithoutEval(Cons(procObj, Cddr(args)), env)
					if forkedErr != nil {
						fmt.Println(forkedErr)
					}
					break Loop
				}
			}
		}, "schedule")
	}()

	return procObj, nil

}

func AbandonImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	procObj := Car(args)

	if !ObjectP(procObj) || ObjectType(procObj) != "Process" {
		err = ProcessError(fmt.Sprintf("adandon expects a Process object expected but received %s.", ObjectType(procObj)), env)
		return
	}

	proc := (*Process)(ObjectValue(procObj))

	if proc.ScheduleTimer == nil {
		return nil, ProcessError("tried to adandon a Process that isn't scheduled", env)
	}

	proc.Abort <- true
	return StringWithValue("OK"), nil
}

func ResetTimeoutImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	procObj := Car(args)

	if !ObjectP(procObj) || ObjectType(procObj) != "Process" {
		err = ProcessError(fmt.Sprintf("restart expects a Process object expected but received %s.", ObjectType(procObj)), env)
		return
	}

	proc := (*Process)(ObjectValue(procObj))

	if proc.ScheduleTimer == nil {
		return nil, ProcessError("tried to reset a Process that isn't scheduled", env)
	}

	var str string
	select {
	case proc.Restart <- true:
		str = "OK"
	default:
		str = "task was already completed or abandoned"
	}
	return StringWithValue(str), nil
}

func JoinImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	procObj := Car(args)

	if !ObjectP(procObj) || ObjectType(procObj) != "Process" {
		err = ProcessError(fmt.Sprintf("join expects a Process object expected but received %s.", ObjectType(procObj)), env)
		return
	}
	proc := (*Process)(ObjectValue(procObj))

	if atomic.CompareAndSwapInt32(&proc.Joined, 0, 1) {
		return <-proc.ReturnValue, nil
	} else {
		return nil, ProcessError("tried to join on a task twice", env)
	}
}

func callWithPanicProtection(f func(), prefix string) {
	defer func() {
		if recovered := recover(); recovered != nil {
			stackBuf := make([]byte, 10000)
			stackBuf = stackBuf[:runtime.Stack(stackBuf, false)]
			stack := strings.Split(string(stackBuf), "\n")
			for i := 0; i < 7; i++ {
				fmt.Println(stack[i])
			}
		}
	}()

	f()
}
<|endoftext|>


<commit_before>package lexer

import (
	"errors"
	"fmt"
	"strings"
	"unicode"
	"unicode/utf8"
)

var errTooManyEmits = errors.New("Too many emits in a single stat function")

// This is returned by next when there are no more characters to read.
const Eof rune = -1

// This is returned when a bad rune is encountered.
const Err rune = utf8.RuneError

// LexInner is the inner type which is used within StateFn to do the actual lexing.
type LexInner struct {
	tokens chan Token
	state  StateFn
	name   string
	input  string
	mark   Mark
	prev   Mark
	async  bool
}

// The Mark type (used by Mark and Unmark) can be used to save
// the current state of the lexer, and restore it later.
type Mark struct {
	pos     int
	line    int
	start   int
	width   int
	replace *Replacer
}

func (mark Mark) rpos() int {
	return mark.pos - mark.start
}

type Replacer struct {
	start, end Mark
	with       string
}

// StateFn is a function that takes a LexInner and returns a StateFn.
type StateFn func(*LexInner) StateFn

// Return the length of the token gathered so far.
func (l *LexInner) Len() int {
	return l.mark.pos - l.mark.start
}

// Get the string of the token gathered so far.
func (l *LexInner) Get() string {
	str := l.input[l.mark.start:l.mark.pos]
	return str
}

// Get the last character accepted into the token.
func (l *LexInner) Last() rune {
	if l.Len() == 0 {
		return Eof
	}
	r, _ := utf8.DecodeLastRuneInString(l.Get())
	return r
}

// Store the state of the lexer.
func (l *LexInner) Mark() Mark {
	return l.mark
}

// Recover the state of the lexer.
func (l *LexInner) Unmark(mark Mark) {
	l.mark = mark
}

// Replace the text from the start Mark to the current position with the given string.
// With may be a different length than the string being replaced, but this change
// will not be reflected by functions like Len and Get.
// Call ReplaceGet to get the token including its replaces. This is how it will be sent by Emit.
// The replace is part of the current Mark, so Unmarking to before a replace was done will
// remove the replace.
func (l *LexInner) Replace(start Mark, with string) {
	if with != "" {
		l.mark.replace = &Replacer{start, l.mark, with}
	}
}

// Get the current token with all replaces included.
// This can be expensive, if you have many replaces.
// Without any replaces, it is identical to Get.
func (l *LexInner) ReplaceGet() string {
	return replaceGet(l.Get(), l.mark.replace)
}

func replaceGet(cur string, replace *Replacer) string {
	if replace == nil {
		return cur
	}
	start := replace.start
	end := replace.end
	return replaceGet(cur[:start.rpos()]+replace.with+cur[end.rpos():], replace.end.replace)
}

// Emit a token with the given type and string.
func (l *LexInner) EmitString(typ TokenType, str string) {
	tok := Token{typ, str, l.name, l.mark.line}
	if l.async {
		l.tokens <- tok
	} else {
		select {
		case l.tokens <- tok:
		default:
			panic(errTooManyEmits)
		}
	}
}

// Emit the gathered token, given its type.
// Emits the result of ReplaceGet, then calls Ignore.
func (l *LexInner) Emit(typ TokenType) {
	l.EmitString(typ, l.ReplaceGet())
	l.Ignore()
}

// Emit a token of type TokenEOF.
// Returns nil.
func (l *LexInner) EmitEof() StateFn {
	l.EmitString(TokenEOF, "EOF")
	return nil
}

// Emit an Error token.
// Like EmitEof, Errorf returns nil.
func (l *LexInner) Errorf(format string, args ...interface{}) StateFn {
	l.EmitString(TokenError, fmt.Sprintf(format, args...))
	return nil
}

// Emit a Warning token.
func (l *LexInner) Warningf(format string, args ...interface{}) {
	l.EmitString(TokenWarning, fmt.Sprintf(format, args...))
}

// Return true if the lexer has reached the end of the file.
func (l *LexInner) Eof() bool {
	if l.mark.pos >= len(l.input) {
		return true
	}
	return false
}

// Read a single character.
// If there are no more characters, it will return Eof.
func (l *LexInner) Next() (char rune) {
	if l.Eof() {
		l.mark.width = 0
		char = Eof
		return Eof
	}
	char, l.mark.width = utf8.DecodeRuneInString(l.input[l.mark.pos:])
	l.mark.pos += l.mark.width
	if char == '\n' {
		l.mark.line++
	}
	return char
}

// Read n characters.
// Returns the number of characters read.
// If it returns less than n, it will have reached EOF.
func (l *LexInner) Skip(n int) int {
	i := 0
	for i = 0; i < n; i++ {
		if l.Next() == Eof {
			return i
		}
	}
	return i
}

// Undo the last Next.
// This is probably won't work after calling any other lexer functions.
// If you need to undo more, use Mark and Unmark.
func (l *LexInner) Back() {
	if l.Last() == '\n' {
		l.mark.line--
	}
	l.mark.pos -= l.mark.width
	l.mark.width = 0
}

// Spy on the upcoming rune.
func (l *LexInner) Peek() rune {
	char := l.Next()
	l.Back()
	return char
}

// Ignore everything gathered about the token so far.
// Also removes any Replaces.
func (l *LexInner) Ignore() {
	l.mark.start = l.mark.pos
	l.mark.width = 0
	l.mark.replace = nil
}

// Retry everything since starting this token.
func (l *LexInner) Retry() {
	l.mark.pos = l.mark.start
	l.mark.width = 0
}

// Attempt to read a string.
// Only if the entire string is successfully accepted does it return true.
// If only a part of the string was matched, none of it is.
func (l *LexInner) String(valid string) bool {
	if strings.HasPrefix(l.input[l.mark.pos:], valid) {
		l.mark.line += strings.Count(valid, "\n")
		l.mark.pos += len(valid)
		l.mark.width = len(valid)
		return true
	}
	return false
}

// Accepts things until the first occurence of the given string.
// The string itself is not accepted.
func (l *LexInner) Find(valid string) bool {
	idx := strings.Index(l.input[l.mark.pos:], valid)
	if idx >= 0 {
		l.mark.line += strings.Count(l.input[l.mark.pos:l.mark.pos+idx], "\n")
		l.mark.pos += idx
		l.mark.width = idx
		return true
	}
	return false
}

// Accept a single character and return true if f returns true.
// Otherwise, do nothing and return false.
func (l *LexInner) One(f func(rune) bool) bool {
	if f(l.Next()) {
		return true
	}
	l.Back()
	return false
}

// Reads characters and feeds them to the given function, and keeps reading until it returns false.
func (l *LexInner) Run(f func(rune) bool) (acceptnum int) {
	for l.One(f) {
		acceptnum++
	}
	return
}

// Accepts any whitespace (unicode.IsSpace), except for whitespace in except.
// For instance, Whitespace("\n") will accept all whitespace except newlines.
// Returns the number of runes read.
func (l *LexInner) Whitespace(except string) (acceptnum int) {
	return l.Run(func(r rune) bool {
		return unicode.IsSpace(r) && !strings.ContainsRune(except, r)
	})
}

func acceptAny(valid string) func(rune) bool {
	return func(char rune) bool {
		return (strings.IndexRune(valid, char) >= 0)
	}
}

// Read one character, but only if it is one of the characters in the given string.
func (l *LexInner) Accept(valid string) bool {
	return l.One(acceptAny(valid))
}

// Read as many characters as possible, but only characters that exist in the given string.
func (l *LexInner) AcceptRun(valid string) (acceptnum int) {
	return l.Run(acceptAny(valid))
}

func not(in func(rune) bool) func(rune) bool {
	return func(char rune) bool {
		if char == Eof {
			return false
		}
		return !in(char)
	}
}

// Read one character, but only if it is NOT one of the characters in the given string.
// If Eof is reached, Except fails regardless of what the given string is.
func (l *LexInner) Except(valid string) bool {
	return l.One(not(acceptAny(valid)))
}

// Read as many characters as possible, but only characters that do NOT exist in the given string.
// If Eof is reached, ExceptRun stops as though it found a successful character.
// Thus, ExceptRun("") accepts everything until Eof.
func (l *LexInner) ExceptRun(valid string) (acceptnum int) {
	return l.Run(not(acceptAny(valid)))
}
<commit_msg>Add docs for Next.<commit_after>package lexer

import (
	"errors"
	"fmt"
	"strings"
	"unicode"
	"unicode/utf8"
)

var errTooManyEmits = errors.New("Too many emits in a single stat function")

// This is returned by next when there are no more characters to read.
const Eof rune = -1

// This is returned when a bad rune is encountered.
const Err rune = utf8.RuneError

// LexInner is the inner type which is used within StateFn to do the actual lexing.
type LexInner struct {
	tokens chan Token
	state  StateFn
	name   string
	input  string
	mark   Mark
	prev   Mark
	async  bool
}

// The Mark type (used by Mark and Unmark) can be used to save
// the current state of the lexer, and restore it later.
type Mark struct {
	pos     int
	line    int
	start   int
	width   int
	replace *Replacer
}

func (mark Mark) rpos() int {
	return mark.pos - mark.start
}

type Replacer struct {
	start, end Mark
	with       string
}

// StateFn is a function that takes a LexInner and returns a StateFn.
type StateFn func(*LexInner) StateFn

// Return the length of the token gathered so far.
func (l *LexInner) Len() int {
	return l.mark.pos - l.mark.start
}

// Get the string of the token gathered so far.
func (l *LexInner) Get() string {
	str := l.input[l.mark.start:l.mark.pos]
	return str
}

// Get the last character accepted into the token.
func (l *LexInner) Last() rune {
	if l.Len() == 0 {
		return Eof
	}
	r, _ := utf8.DecodeLastRuneInString(l.Get())
	return r
}

// Store the state of the lexer.
func (l *LexInner) Mark() Mark {
	return l.mark
}

// Recover the state of the lexer.
func (l *LexInner) Unmark(mark Mark) {
	l.mark = mark
}

// Replace the text from the start Mark to the current position with the given string.
// With may be a different length than the string being replaced, but this change
// will not be reflected by functions like Len and Get.
// Call ReplaceGet to get the token including its replaces. This is how it will be sent by Emit.
// The replace is part of the current Mark, so Unmarking to before a replace was done will
// remove the replace.
func (l *LexInner) Replace(start Mark, with string) {
	if with != "" {
		l.mark.replace = &Replacer{start, l.mark, with}
	}
}

// Get the current token with all replaces included.
// This can be expensive, if you have many replaces.
// Without any replaces, it is identical to Get.
func (l *LexInner) ReplaceGet() string {
	return replaceGet(l.Get(), l.mark.replace)
}

func replaceGet(cur string, replace *Replacer) string {
	if replace == nil {
		return cur
	}
	start := replace.start
	end := replace.end
	return replaceGet(cur[:start.rpos()]+replace.with+cur[end.rpos():], replace.end.replace)
}

// Emit a token with the given type and string.
func (l *LexInner) EmitString(typ TokenType, str string) {
	tok := Token{typ, str, l.name, l.mark.line}
	if l.async {
		l.tokens <- tok
	} else {
		select {
		case l.tokens <- tok:
		default:
			panic(errTooManyEmits)
		}
	}
}

// Emit the gathered token, given its type.
// Emits the result of ReplaceGet, then calls Ignore.
func (l *LexInner) Emit(typ TokenType) {
	l.EmitString(typ, l.ReplaceGet())
	l.Ignore()
}

// Emit a token of type TokenEOF.
// Returns nil.
func (l *LexInner) EmitEof() StateFn {
	l.EmitString(TokenEOF, "EOF")
	return nil
}

// Emit an Error token.
// Like EmitEof, Errorf returns nil.
func (l *LexInner) Errorf(format string, args ...interface{}) StateFn {
	l.EmitString(TokenError, fmt.Sprintf(format, args...))
	return nil
}

// Emit a Warning token.
func (l *LexInner) Warningf(format string, args ...interface{}) {
	l.EmitString(TokenWarning, fmt.Sprintf(format, args...))
}

// Return true if the lexer has reached the end of the file.
func (l *LexInner) Eof() bool {
	if l.mark.pos >= len(l.input) {
		return true
	}
	return false
}

// Read a single character.
// If there are no more characters, it will return Eof.
// If a non-utf8 character is read, it will return Err.
func (l *LexInner) Next() (char rune) {
	if l.Eof() {
		l.mark.width = 0
		char = Eof
		return Eof
	}
	char, l.mark.width = utf8.DecodeRuneInString(l.input[l.mark.pos:])
	l.mark.pos += l.mark.width
	if char == '\n' {
		l.mark.line++
	}
	return char
}

// Read n characters.
// Returns the number of characters read.
// If it returns less than n, it will have reached EOF.
func (l *LexInner) Skip(n int) int {
	i := 0
	for i = 0; i < n; i++ {
		if l.Next() == Eof {
			return i
		}
	}
	return i
}

// Undo the last Next.
// This is probably won't work after calling any other lexer functions.
// If you need to undo more, use Mark and Unmark.
func (l *LexInner) Back() {
	if l.Last() == '\n' {
		l.mark.line--
	}
	l.mark.pos -= l.mark.width
	l.mark.width = 0
}

// Spy on the upcoming rune.
func (l *LexInner) Peek() rune {
	char := l.Next()
	l.Back()
	return char
}

// Ignore everything gathered about the token so far.
// Also removes any Replaces.
func (l *LexInner) Ignore() {
	l.mark.start = l.mark.pos
	l.mark.width = 0
	l.mark.replace = nil
}

// Retry everything since starting this token.
func (l *LexInner) Retry() {
	l.mark.pos = l.mark.start
	l.mark.width = 0
}

// Attempt to read a string.
// Only if the entire string is successfully accepted does it return true.
// If only a part of the string was matched, none of it is.
func (l *LexInner) String(valid string) bool {
	if strings.HasPrefix(l.input[l.mark.pos:], valid) {
		l.mark.line += strings.Count(valid, "\n")
		l.mark.pos += len(valid)
		l.mark.width = len(valid)
		return true
	}
	return false
}

// Accepts things until the first occurence of the given string.
// The string itself is not accepted.
func (l *LexInner) Find(valid string) bool {
	idx := strings.Index(l.input[l.mark.pos:], valid)
	if idx >= 0 {
		l.mark.line += strings.Count(l.input[l.mark.pos:l.mark.pos+idx], "\n")
		l.mark.pos += idx
		l.mark.width = idx
		return true
	}
	return false
}

// Accept a single character and return true if f returns true.
// Otherwise, do nothing and return false.
func (l *LexInner) One(f func(rune) bool) bool {
	if f(l.Next()) {
		return true
	}
	l.Back()
	return false
}

// Reads characters and feeds them to the given function, and keeps reading until it returns false.
func (l *LexInner) Run(f func(rune) bool) (acceptnum int) {
	for l.One(f) {
		acceptnum++
	}
	return
}

// Accepts any whitespace (unicode.IsSpace), except for whitespace in except.
// For instance, Whitespace("\n") will accept all whitespace except newlines.
// Returns the number of runes read.
func (l *LexInner) Whitespace(except string) (acceptnum int) {
	return l.Run(func(r rune) bool {
		return unicode.IsSpace(r) && !strings.ContainsRune(except, r)
	})
}

func acceptAny(valid string) func(rune) bool {
	return func(char rune) bool {
		return (strings.IndexRune(valid, char) >= 0)
	}
}

// Read one character, but only if it is one of the characters in the given string.
func (l *LexInner) Accept(valid string) bool {
	return l.One(acceptAny(valid))
}

// Read as many characters as possible, but only characters that exist in the given string.
func (l *LexInner) AcceptRun(valid string) (acceptnum int) {
	return l.Run(acceptAny(valid))
}

func not(in func(rune) bool) func(rune) bool {
	return func(char rune) bool {
		if char == Eof {
			return false
		}
		return !in(char)
	}
}

// Read one character, but only if it is NOT one of the characters in the given string.
// If Eof is reached, Except fails regardless of what the given string is.
func (l *LexInner) Except(valid string) bool {
	return l.One(not(acceptAny(valid)))
}

// Read as many characters as possible, but only characters that do NOT exist in the given string.
// If Eof is reached, ExceptRun stops as though it found a successful character.
// Thus, ExceptRun("") accepts everything until Eof.
func (l *LexInner) ExceptRun(valid string) (acceptnum int) {
	return l.Run(not(acceptAny(valid)))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"encoding/gob"
	"log"
	"os"
	"time"

	"github.com/szferi/gomdb"
)

type Storage struct {
	env  *mdb.Env
	dbi  *mdb.DBI
	send AmqpSender

	timerRunning bool
	timer        *time.Timer
	nextSend     time.Time
}

func NewStorage(send AmqpSender) (s *Storage, err error) {
	s = new(Storage)
	s.send = send

	s.env, err = mdb.NewEnv()
	if err != nil {
		return
	}

	storageDir := "db"
	err = os.MkdirAll(storageDir, 0755)
	if err != nil {
		return
	}

	err = s.env.Open(storageDir, 0, 0755)
	if err != nil {
		return
	}

	s.timerRunning = true
	s.timer = time.NewTimer(time.Duration(3) * time.Second)
	s.nextSend = time.Now().Add(time.Duration(3) * time.Second)
	go func() {
		// XXX change the timer to run a SEND command through raft, and emit on applying that command.
		t := <-s.timer.C
		entries, err := s.get(t)
		// XXX cleanup and shut down properly
		if err != nil {
			log.Fatal("Could not read entries from db: ", err)
		}

		log.Printf("Sending %d entries\n", len(entries))
		for _, e := range entries {
			s.send.C <- e
		}
	}()

	return
}

func (s *Storage) Close() {
	s.env.Close()
	s.timer.Stop()
}

func (s *Storage) Add(e Entry, r []byte) error {
	txn, err := s.env.BeginTxn(nil, 0)
	if err != nil {
		return err
	}

	dbi, err := txn.DBIOpen(nil, 0)
	if err != nil {
		return err
	}

	k := uint64ToBytes(uint64(e.SendAt.UnixNano()))
	v := r

	txn.Put(dbi, k, v, 0)

	if !s.timerRunning || e.SendAt.Before(s.nextSend) {
		s.timer.Reset(time.Now().Sub(e.SendAt))
		s.nextSend = e.SendAt
		s.timerRunning = true
	}

	txn.Commit()

	return nil
}

func (s *Storage) get(t time.Time) (entries []Entry, err error) {
	txn, err := s.env.BeginTxn(nil, mdb.RDONLY)
	if err != nil {
		log.Println("Error creating transaction: ", err)
		return
	}
	defer txn.Abort()

	dbi, err := txn.DBIOpen(nil, 0)
	if err != nil {
		log.Println("Error opening dbi: ", err)
		return
	}
	defer s.env.DBIClose(dbi)

	cursor, err := txn.CursorOpen(dbi)
	if err != nil {
		log.Println("Error getting cursor: ", err)
		return
	}
	defer cursor.Close()

	sk := uint64(t.UnixNano())
	log.Println("Looking for: ", t, t.UnixNano())

	for {
		k, v, err := cursor.Get(nil, mdb.NEXT)
		if err == mdb.NotFound {
			break
		}
		if err != nil {
			panic(err)
		}

		kt := bytesToUint64(k)
		if kt > sk {
			break
		}

		entry := Entry{}
		dec := gob.NewDecoder(bytes.NewBuffer(v))
		err = dec.Decode(&entry)
		if err != nil {
			panic(err)
		}

		entries = append(entries, entry)
	}

	return
}

// Converts bytes to an integer
func bytesToUint64(b []byte) uint64 {
	return binary.BigEndian.Uint64(b)
}

// Converts a uint to a byte slice
func uint64ToBytes(u uint64) []byte {
	buf := make([]byte, 8)
	binary.BigEndian.PutUint64(buf, u)
	return buf
}
<commit_msg>Read initial send time from db, and reset after send<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"encoding/gob"
	"log"
	"os"
	"time"

	"github.com/szferi/gomdb"
)

type Storage struct {
	env  *mdb.Env
	dbi  *mdb.DBI
	send AmqpSender

	timerRunning bool
	timer        *time.Timer
	nextSend     time.Time
}

func NewStorage(send AmqpSender) (s *Storage, err error) {
	s = new(Storage)
	s.send = send

	s.env, err = mdb.NewEnv()
	if err != nil {
		return
	}

	storageDir := "db"
	err = os.MkdirAll(storageDir, 0755)
	if err != nil {
		return
	}

	err = s.env.Open(storageDir, 0, 0755)
	if err != nil {
		return
	}

	t, err := s.nextTime()
	if err != nil {
		return
	}

	s.timer = time.NewTimer(time.Duration(24) * time.Hour)
	s.resetTimer(t)

	go func() {
		for {
			// XXX change the timer to run a SEND command through raft, and emit on applying that command.
			t := <-s.timer.C
			entries, err := s.get(t)
			// XXX cleanup and shut down properly
			if err != nil {
				log.Fatal("Could not read entries from db: ", err)
			}

			log.Printf("Sending %d entries\n", len(entries))
			for _, e := range entries {
				s.send.C <- e
			}

			t, err = s.nextTime()
			if err != nil {
				log.Fatal("Could not read next time from db: ", err)
			}
			s.resetTimer(t)
		}
	}()

	return
}

func (s *Storage) Close() {
	s.env.Close()
	s.timer.Stop()
}

func (s *Storage) Add(e Entry, r []byte) error {
	txn, err := s.env.BeginTxn(nil, 0)
	if err != nil {
		return err
	}

	dbi, err := txn.DBIOpen(nil, 0)
	if err != nil {
		return err
	}

	k := uint64ToBytes(uint64(e.SendAt.UnixNano()))
	v := r

	txn.Put(dbi, k, v, 0)

	if !s.timerRunning || e.SendAt.Before(s.nextSend) {
		s.resetTimer(e.SendAt)
	}

	txn.Commit()

	return nil
}

func (s *Storage) get(t time.Time) (entries []Entry, err error) {
	txn, err := s.env.BeginTxn(nil, mdb.RDONLY)
	if err != nil {
		log.Println("Error creating transaction: ", err)
		return
	}
	defer txn.Abort()

	dbi, err := txn.DBIOpen(nil, 0)
	if err != nil {
		log.Println("Error opening dbi: ", err)
		return
	}
	defer s.env.DBIClose(dbi)

	cursor, err := txn.CursorOpen(dbi)
	if err != nil {
		log.Println("Error getting cursor: ", err)
		return
	}
	defer cursor.Close()

	sk := uint64(t.UnixNano())
	log.Println("Looking for: ", t, t.UnixNano())

	for {
		k, v, err := cursor.Get(nil, mdb.NEXT)
		if err == mdb.NotFound {
			break
		}
		if err != nil {
			panic(err)
		}

		kt := bytesToUint64(k)
		if kt > sk {
			break
		}

		entry := Entry{}
		dec := gob.NewDecoder(bytes.NewBuffer(v))
		err = dec.Decode(&entry)
		if err != nil {
			panic(err)
		}

		entries = append(entries, entry)
	}

	return
}

// get the next entry send time from the db
func (s *Storage) nextTime() (t time.Time, err error) {
	txn, err := s.env.BeginTxn(nil, mdb.RDONLY)
	if err != nil {
		log.Println("Error creating transaction: ", err)
		return
	}
	defer txn.Abort()

	dbi, err := txn.DBIOpen(nil, 0)
	if err != nil {
		log.Println("Error opening dbi: ", err)
		return
	}
	defer s.env.DBIClose(dbi)

	cursor, err := txn.CursorOpen(dbi)
	if err != nil {
		log.Println("Error getting cursor: ", err)
		return
	}
	defer cursor.Close()

	k, _, err := cursor.Get(nil, mdb.FIRST)
	if err != nil {
		log.Println("Error reading next time from db: ", err)
		return
	}

	t = time.Unix(0, int64(bytesToUint64(k)))

	return
}

func (s *Storage) resetTimer(t time.Time) {
	if t.Before(time.Now()) {
		t = time.Now()
	}
	log.Println("Setting next timer for: ", t)
	s.timerRunning = true
	s.timer.Reset(t.Sub(time.Now()))
	s.nextSend = t
}

// Converts bytes to an integer
func bytesToUint64(b []byte) uint64 {
	return binary.BigEndian.Uint64(b)
}

// Converts a uint to a byte slice
func uint64ToBytes(u uint64) []byte {
	buf := make([]byte, 8)
	binary.BigEndian.PutUint64(buf, u)
	return buf
}
<|endoftext|>


<commit_before>package classification

import (
	"context"
	"fmt"

	"github.com/go-openapi/strfmt"
	"github.com/semi-technologies/weaviate/entities/filters"
	"github.com/semi-technologies/weaviate/entities/models"
	"github.com/semi-technologies/weaviate/entities/schema"
	"github.com/semi-technologies/weaviate/entities/schema/crossref"
	"github.com/semi-technologies/weaviate/entities/schema/kind"
	"github.com/semi-technologies/weaviate/entities/search"
	"github.com/semi-technologies/weaviate/usecases/traverser"
)

func (c *Classifier) classifyItemContextual(item search.Result, kind kind.Kind, params models.Classification) error {
	s := c.schemaGetter.GetSchemaSkipAuth()
	// safe to skip nil-check as we passed validation

	var classified []string
	for _, propName := range params.ClassifyProperties {
		prop, err := s.GetProperty(kind, schema.ClassName(params.Class), schema.PropertyName(propName))
		if err != nil {
			return fmt.Errorf("contextual: get target prop '%s': %v", propName, err)
		}

		dataType, err := s.FindPropertyDataType(prop.DataType)
		if err != nil {
			return fmt.Errorf("contextual: extract dataType of prop '%s': %v", propName, err)
		}

		// we have passed validation, so it is safe to assume that this is a ref prop
		targetClasses := dataType.Classes()

		// len=1 is guaranteed from validation
		targetClass := targetClasses[0]
		targetKind, _ := s.GetKindOfClass(targetClass)

		res, err := c.vectorRepo.VectorClassSearch(context.Background(), traverser.GetParams{
			SearchVector: item.Vector,
			ClassName:    targetClass.String(),
			Kind:         targetKind,
			Pagination: &filters.Pagination{
				Limit: 1,
			},
			Properties: traverser.SelectProperties{
				traverser.SelectProperty{
					Name: "uuid",
				},
			},
		})
		if err != nil {
			return fmt.Errorf("contextual: search closest target: %v", err)
		}

		if res == nil || len(res) == 0 {
			return fmt.Errorf("contexual: no potential targets found of class '%s' (%s)", targetClass, targetKind)
		}

		targetBeacon := crossref.New("localhost", res[0].ID, res[0].Kind).String()
		item.Schema.(map[string]interface{})[propName] = models.MultipleRef{
			&models.SingleRef{
				Beacon: strfmt.URI(targetBeacon),
				Meta: &models.ReferenceMeta{
					Classification: &models.ReferenceMetaClassification{
						WinningDistance: float64(9000), // TODO
					},
				},
			},
		}

		// append list of actually classified (can differ from scope!) properties,
		// so we can build the object meta information
		classified = append(classified, propName)
	}

	c.extendItemWithObjectMeta(&item, params, classified)
	err := c.store(item)
	if err != nil {
		return fmt.Errorf("store %s/%s: %v", item.ClassName, item.ID, err)
	}

	return nil
}
<commit_msg>gh-1033 clean up classify item contextually<commit_after>package classification

import (
	"context"
	"fmt"

	"github.com/go-openapi/strfmt"
	"github.com/semi-technologies/weaviate/entities/filters"
	"github.com/semi-technologies/weaviate/entities/models"
	"github.com/semi-technologies/weaviate/entities/schema"
	"github.com/semi-technologies/weaviate/entities/schema/crossref"
	"github.com/semi-technologies/weaviate/entities/schema/kind"
	"github.com/semi-technologies/weaviate/entities/search"
	"github.com/semi-technologies/weaviate/usecases/traverser"
)

type contextualItemClassifier struct {
	item       search.Result
	kind       kind.Kind
	params     models.Classification
	classifier *Classifier
	schema     schema.Schema
}

func (c *Classifier) classifyItemContextual(item search.Result, kind kind.Kind, params models.Classification) error {
	schema := c.schemaGetter.GetSchemaSkipAuth()
	run := &contextualItemClassifier{
		item:       item,
		kind:       kind,
		params:     params,
		classifier: c,
		schema:     schema,
	}

	err := run.do()
	if err != nil {
		return fmt.Errorf("contextual: %v", err)
	}

	return nil
}

func (c *contextualItemClassifier) do() error {
	var classified []string
	for _, propName := range c.params.ClassifyProperties {
		current, err := c.property(propName)
		if err != nil {
			return fmt.Errorf("prop '%s': %v", propName, err)
		}

		// append list of actually classified (can differ from scope!) properties,
		// so we can build the object meta information
		classified = append(classified, current)
	}

	c.classifier.extendItemWithObjectMeta(&c.item, c.params, classified)
	err := c.classifier.store(c.item)
	if err != nil {
		return fmt.Errorf("store %s/%s: %v", c.item.ClassName, c.item.ID, err)
	}

	return nil
}

func (c *contextualItemClassifier) property(propName string) (string, error) {
	targetClass, targetKind, err := c.classAndKindOfTarget(propName)
	if err != nil {
		return "", fmt.Errorf("inspect target: %v", err)
	}

	res, err := c.findTarget(targetClass, targetKind)
	if err != nil {
		return "", fmt.Errorf("find target: %v", err)
	}

	targetBeacon := crossref.New("localhost", res.ID, res.Kind).String()
	c.item.Schema.(map[string]interface{})[propName] = models.MultipleRef{
		&models.SingleRef{
			Beacon: strfmt.URI(targetBeacon),
			Meta: &models.ReferenceMeta{
				Classification: &models.ReferenceMetaClassification{
					WinningDistance: float64(9000), // TODO
				},
			},
		},
	}

	return propName, nil
}

func (c *contextualItemClassifier) classAndKindOfTarget(propName string) (schema.ClassName, kind.Kind, error) {
	prop, err := c.schema.GetProperty(c.kind, schema.ClassName(c.params.Class), schema.PropertyName(propName))
	if err != nil {
		return "", "", fmt.Errorf("get target prop '%s': %v", propName, err)
	}

	dataType, err := c.schema.FindPropertyDataType(prop.DataType)
	if err != nil {
		return "", "", fmt.Errorf("extract dataType of prop '%s': %v", propName, err)
	}

	// we have passed validation, so it is safe to assume that this is a ref prop
	targetClasses := dataType.Classes()

	// len=1 is guaranteed from validation
	targetClass := targetClasses[0]
	targetKind, _ := c.schema.GetKindOfClass(targetClass)

	return targetClass, targetKind, nil
}

func (c *contextualItemClassifier) findTarget(targetClass schema.ClassName, targetKind kind.Kind) (*search.Result, error) {
	res, err := c.classifier.vectorRepo.VectorClassSearch(context.Background(), traverser.GetParams{
		SearchVector: c.item.Vector,
		ClassName:    targetClass.String(),
		Kind:         targetKind,
		Pagination: &filters.Pagination{
			Limit: 1,
		},
		Properties: traverser.SelectProperties{
			traverser.SelectProperty{
				Name: "uuid",
			},
		},
	})
	if err != nil {
		return nil, fmt.Errorf("search closest target: %v", err)
	}

	if res == nil || len(res) == 0 {
		return nil, fmt.Errorf("no potential targets found of class '%s' (%s)", targetClass, targetKind)
	}

	return &res[0], nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Company 0, LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package zkutil

import (
	"bytes"
	"fmt"
	"os/user"
	"path"
	"strings"
)

// semanticAlphabet
const semanticAlphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-"

// These constants define the application version and follow the semantic
// versioning 2.0.0 spec (http://semver.org/).
const (
	appMajor uint = 0
	appMinor uint = 3
	appPatch uint = 0

	// appPreRelease MUST only contain characters from semanticAlphabet
	// per the semantic versioning spec.
	appPreRelease = ""
)

// appBuild is defined as a variable so it can be overridden during the build
// process with '-ldflags "-X main.appBuild foo' if needed.  It MUST only
// contain characters from semanticAlphabet per the semantic versioning spec.
var appBuild string

// Version returns the application version as a properly formed string per the
// semantic versioning 2.0.0 spec (http://semver.org/).
func Version() string {
	// Start with the major, minor, and patch versions.
	version := fmt.Sprintf("%d.%d.%d", appMajor, appMinor, appPatch)

	// Append pre-release version if there is one.  The hyphen called for
	// by the semantic versioning spec is automatically appended and should
	// not be contained in the pre-release string.  The pre-release version
	// is not appended if it contains invalid characters.
	preRelease := normalizeVerString(appPreRelease)
	if preRelease != "" {
		version = fmt.Sprintf("%s-%s", version, preRelease)
	}

	// Append build metadata if there is any.  The plus called for
	// by the semantic versioning spec is automatically appended and should
	// not be contained in the build metadata string.  The build metadata
	// string is not appended if it contains invalid characters.
	build := normalizeVerString(appBuild)
	if build != "" {
		version = fmt.Sprintf("%s+%s", version, build)
	}

	return version
}

// normalizeVerString returns the passed string stripped of all characters which
// are not valid according to the semantic versioning guidelines for pre-release
// version and build metadata strings.  In particular they MUST only contain
// characters in semanticAlphabet.
func normalizeVerString(str string) string {
	var result bytes.Buffer
	for _, r := range str {
		if strings.ContainsRune(semanticAlphabet, r) {
			result.WriteRune(r)
		}
	}
	return result.String()
}

const (
	DefaultZKClientDir  = ".zkclient"
	DefaultZKServerDir  = ".zkserver"
	DefaultZKClientLog  = "zkclient.log"
	DefaultZKClientConf = "zkclient.conf"
)

func DefaultClientRootPath() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", fmt.Errorf("user.Current: %v", err)
	}
	return path.Join(usr.HomeDir, DefaultZKClientDir), nil
}

func DefaultServerRootPath() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", fmt.Errorf("user.Current: %v", err)
	}
	return path.Join(usr.HomeDir, DefaultZKServerDir), nil
}
<commit_msg>bump version (#141)<commit_after>// Copyright (c) 2016 Company 0, LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package zkutil

import (
	"bytes"
	"fmt"
	"os/user"
	"path"
	"strings"
)

// semanticAlphabet
const semanticAlphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-"

// These constants define the application version and follow the semantic
// versioning 2.0.0 spec (http://semver.org/).
const (
	appMajor uint = 0
	appMinor uint = 3
	appPatch uint = 1

	// appPreRelease MUST only contain characters from semanticAlphabet
	// per the semantic versioning spec.
	appPreRelease = ""
)

// appBuild is defined as a variable so it can be overridden during the build
// process with '-ldflags "-X main.appBuild foo' if needed.  It MUST only
// contain characters from semanticAlphabet per the semantic versioning spec.
var appBuild string

// Version returns the application version as a properly formed string per the
// semantic versioning 2.0.0 spec (http://semver.org/).
func Version() string {
	// Start with the major, minor, and patch versions.
	version := fmt.Sprintf("%d.%d.%d", appMajor, appMinor, appPatch)

	// Append pre-release version if there is one.  The hyphen called for
	// by the semantic versioning spec is automatically appended and should
	// not be contained in the pre-release string.  The pre-release version
	// is not appended if it contains invalid characters.
	preRelease := normalizeVerString(appPreRelease)
	if preRelease != "" {
		version = fmt.Sprintf("%s-%s", version, preRelease)
	}

	// Append build metadata if there is any.  The plus called for
	// by the semantic versioning spec is automatically appended and should
	// not be contained in the build metadata string.  The build metadata
	// string is not appended if it contains invalid characters.
	build := normalizeVerString(appBuild)
	if build != "" {
		version = fmt.Sprintf("%s+%s", version, build)
	}

	return version
}

// normalizeVerString returns the passed string stripped of all characters which
// are not valid according to the semantic versioning guidelines for pre-release
// version and build metadata strings.  In particular they MUST only contain
// characters in semanticAlphabet.
func normalizeVerString(str string) string {
	var result bytes.Buffer
	for _, r := range str {
		if strings.ContainsRune(semanticAlphabet, r) {
			result.WriteRune(r)
		}
	}
	return result.String()
}

const (
	DefaultZKClientDir  = ".zkclient"
	DefaultZKServerDir  = ".zkserver"
	DefaultZKClientLog  = "zkclient.log"
	DefaultZKClientConf = "zkclient.conf"
)

func DefaultClientRootPath() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", fmt.Errorf("user.Current: %v", err)
	}
	return path.Join(usr.HomeDir, DefaultZKClientDir), nil
}

func DefaultServerRootPath() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", fmt.Errorf("user.Current: %v", err)
	}
	return path.Join(usr.HomeDir, DefaultZKServerDir), nil
}
<|endoftext|>