<commit_before>package store

import (
	"encoding/json"
	"fmt"
	"path"
	"sort"
	"strings"
	"sync"
	"time"

	etcdErr "github.com/coreos/etcd/error"
)

type Store struct {
	Root       *Node
	WatcherHub *watcherHub
	Index      uint64
	Term       uint64
	Stats      *Stats
	worldLock  sync.RWMutex // stop the world lock. Used to do snapshot
}

func New() *Store {
	s := new(Store)
	s.Root = newDir("/", 0, 0, nil, "", Permanent)
	s.Stats = newStats()
	s.WatcherHub = newWatchHub(1000)

	return s
}

func (s *Store) Get(nodePath string, recursive, sorted bool, index uint64, term uint64) (*Event, error) {
	s.worldLock.RLock()
	defer s.worldLock.RUnlock()

	nodePath = path.Clean(path.Join("/", nodePath))

	n, err := s.internalGet(nodePath, index, term)

	if err != nil {
		s.Stats.Inc(GetFail)
		return nil, err
	}

	e := newEvent(Get, nodePath, index, term)

	if n.IsDir() { // node is dir
		e.Dir = true

		children, _ := n.List()
		e.KVPairs = make([]KeyValuePair, len(children))

		// we do not use the index in the children slice directly
		// we need to skip the hidden one
		i := 0

		for _, child := range children {

			if child.IsHidden() { // get will not list hidden node
				continue
			}

			e.KVPairs[i] = child.Pair(recursive, sorted)

			i++
		}

		// eliminate hidden nodes
		e.KVPairs = e.KVPairs[:i]

		rootPairs := KeyValuePair{
			KVPairs: e.KVPairs,
		}

		if sorted {
			sort.Sort(rootPairs)
		}

	} else { // node is file
		e.Value = n.Value
	}

	if n.ExpireTime.Sub(Permanent) != 0 {
		e.Expiration = &n.ExpireTime
		e.TTL = int64(n.ExpireTime.Sub(time.Now())/time.Second) + 1
	}

	s.Stats.Inc(GetSuccess)

	return e, nil
}

// Create function creates the Node at nodePath. Create will help to create intermediate directories with no ttl.
// If the node has already existed, create will fail.
// If any node on the path is a file, create will fail.
func (s *Store) Create(nodePath string, value string, expireTime time.Time, index uint64, term uint64) (*Event, error) {
	s.worldLock.RLock()
	defer s.worldLock.RUnlock()

	nodePath = path.Clean(path.Join("/", nodePath))

	// make sure we can create the node
	_, err := s.internalGet(nodePath, index, term)

	if err == nil { // key already exists
		s.Stats.Inc(SetFail)
		return nil, etcdErr.NewError(etcdErr.EcodeNodeExist, nodePath)
	}

	etcdError, _ := err.(etcdErr.Error)

	if etcdError.ErrorCode == 104 { // we cannot create the key due to meet a file while walking
		s.Stats.Inc(SetFail)
		return nil, err
	}

	dir, _ := path.Split(nodePath)

	// walk through the nodePath, create dirs and get the last directory node
	d, err := s.walk(dir, s.checkDir)

	if err != nil {
		s.Stats.Inc(SetFail)
		return nil, err
	}

	e := newEvent(Create, nodePath, s.Index, s.Term)

	var n *Node

	if len(value) != 0 { // create file
		e.Value = value

		n = newFile(nodePath, value, s.Index, s.Term, d, "", expireTime)

	} else { // create directory
		e.Dir = true

		n = newDir(nodePath, s.Index, s.Term, d, "", expireTime)

	}

	err = d.Add(n)

	if err != nil {
		s.Stats.Inc(SetFail)
		return nil, err
	}

	// Node with TTL
	if expireTime.Sub(Permanent) != 0 {
		n.Expire()
		e.Expiration = &n.ExpireTime
		e.TTL = int64(expireTime.Sub(time.Now())/time.Second) + 1
	}

	s.WatcherHub.notify(e)
	s.Stats.Inc(SetSuccess)
	return e, nil
}

// Update function updates the value/ttl of the node.
// If the node is a file, the value and the ttl can be updated.
// If the node is a directory, only the ttl can be updated.
func (s *Store) Update(nodePath string, value string, expireTime time.Time, index uint64, term uint64) (*Event, error) {
	s.worldLock.RLock()
	defer s.worldLock.RUnlock()

	n, err := s.internalGet(nodePath, index, term)

	if err != nil { // if the node does not exist, return error
		s.Stats.Inc(UpdateFail)
		return nil, err
	}

	e := newEvent(Update, nodePath, s.Index, s.Term)

	if n.IsDir() { // if the node is a directory, we can only update ttl

		if len(value) != 0 {
			s.Stats.Inc(UpdateFail)
			return nil, etcdErr.NewError(etcdErr.EcodeNotFile, nodePath)
		}

	} else { // if the node is a file, we can update value and ttl
		e.PrevValue = n.Value

		if len(value) != 0 {
			e.Value = value
		}

		n.Write(value, index, term)
	}

	// update ttl
	if !n.IsPermanent() {
		n.stopExpire <- true
	}

	if expireTime.Sub(Permanent) != 0 {
		n.ExpireTime = expireTime
		n.Expire()
		e.Expiration = &n.ExpireTime
		e.TTL = int64(expireTime.Sub(time.Now())/time.Second) + 1
	}

	s.WatcherHub.notify(e)
	s.Stats.Inc(UpdateSuccess)
	return e, nil
}

func (s *Store) TestAndSet(nodePath string, prevValue string, prevIndex uint64,
	value string, expireTime time.Time, index uint64, term uint64) (*Event, error) {

	s.worldLock.RLock()
	defer s.worldLock.RUnlock()

	f, err := s.internalGet(nodePath, index, term)

	if err != nil {
		s.Stats.Inc(TestAndSetFail)
		return nil, err
	}

	if f.IsDir() { // can only test and set file
		s.Stats.Inc(TestAndSetFail)
		return nil, etcdErr.NewError(etcdErr.EcodeNotFile, nodePath)
	}

	if f.Value == prevValue || f.ModifiedIndex == prevIndex {
		// if test succeed, write the value
		e := newEvent(TestAndSet, nodePath, index, term)
		e.PrevValue = f.Value
		e.Value = value
		f.Write(value, index, term)

		s.WatcherHub.notify(e)
		s.Stats.Inc(TestAndSetSuccess)
		return e, nil
	}

	cause := fmt.Sprintf("[%v != %v] [%v != %v]", prevValue, f.Value, prevIndex, f.ModifiedIndex)
	s.Stats.Inc(TestAndSetFail)
	return nil, etcdErr.NewError(etcdErr.EcodeTestFailed, cause)
}

// Delete function deletes the node at the given path.
// If the node is a directory, recursive must be true to delete it.
func (s *Store) Delete(nodePath string, recursive bool, index uint64, term uint64) (*Event, error) {
	s.worldLock.RLock()
	defer s.worldLock.RUnlock()

	n, err := s.internalGet(nodePath, index, term)

	if err != nil { // if the node does not exist, return error
		s.Stats.Inc(DeleteFail)
		return nil, err
	}

	e := newEvent(Delete, nodePath, index, term)

	if n.IsDir() {
		e.Dir = true
	} else {
		e.PrevValue = n.Value
	}

	callback := func(path string) { // notify function
		s.WatcherHub.notifyWithPath(e, path, true)
	}

	err = n.Remove(recursive, callback)

	if err != nil {
		s.Stats.Inc(DeleteFail)
		return nil, err
	}

	s.WatcherHub.notify(e)
	s.Stats.Inc(DeleteSuccess)

	return e, nil
}

func (s *Store) Watch(prefix string, recursive bool, sinceIndex uint64, index uint64, term uint64) (<-chan *Event, error) {
	s.worldLock.RLock()
	defer s.worldLock.RUnlock()

	s.Index, s.Term = index, term

	if sinceIndex == 0 {
		return s.WatcherHub.watch(prefix, recursive, index+1)
	}

	return s.WatcherHub.watch(prefix, recursive, sinceIndex)
}

// walk function walks all the nodePath and apply the walkFunc on each directory
func (s *Store) walk(nodePath string, walkFunc func(prev *Node, component string) (*Node, error)) (*Node, error) {
	components := strings.Split(nodePath, "/")

	curr := s.Root

	var err error
	for i := 1; i < len(components); i++ {
		if len(components[i]) == 0 { // ignore empty string
			return curr, nil
		}

		curr, err = walkFunc(curr, components[i])
		if err != nil {
			return nil, err
		}

	}

	return curr, nil
}

// InternalGet function get the node of the given nodePath.
func (s *Store) internalGet(nodePath string, index uint64, term uint64) (*Node, error) {
	nodePath = path.Clean(path.Join("/", nodePath))

	// update file system known index and term
	s.Index, s.Term = index, term

	walkFunc := func(parent *Node, name string) (*Node, error) {

		if !parent.IsDir() {
			return nil, etcdErr.NewError(etcdErr.EcodeNotDir, parent.Path)
		}

		child, ok := parent.Children[name]
		if ok {
			return child, nil
		}

		return nil, etcdErr.NewError(etcdErr.EcodeKeyNotFound, path.Join(parent.Path, name))
	}

	f, err := s.walk(nodePath, walkFunc)

	if err != nil {
		return nil, err
	}

	return f, nil
}

// checkDir function will check whether the component is a directory under parent node.
// If it is a directory, this function will return the pointer to that node.
// If it does not exist, this function will create a new directory and return the pointer to that node.
// If it is a file, this function will return error.
func (s *Store) checkDir(parent *Node, dirName string) (*Node, error) {
	subDir, ok := parent.Children[dirName]

	if ok {
		return subDir, nil
	}

	n := newDir(path.Join(parent.Path, dirName), s.Index, s.Term, parent, parent.ACL, Permanent)

	parent.Children[dirName] = n

	return n, nil
}

// Save function saves the static state of the store system.
// Save function will not be able to save the state of watchers.
// Save function will not save the parent field of the node. Or there will
// be cyclic dependencies issue for the json package.
func (s *Store) Save() ([]byte, error) {
	s.worldLock.Lock()

	clonedStore := New()
	clonedStore.Root = s.Root.Clone()
	clonedStore.WatcherHub = s.WatcherHub.clone()
	clonedStore.Index = s.Index
	clonedStore.Term = s.Term
	clonedStore.Stats = s.Stats.clone()

	s.worldLock.Unlock()

	b, err := json.Marshal(clonedStore)

	if err != nil {
		return nil, err
	}

	return b, nil
}

// recovery function recovery the store system from a static state.
// It needs to recovery the parent field of the nodes.
// It needs to delete the expired nodes since the saved time and also
// need to create monitor go routines.
func (s *Store) Recovery(state []byte) error {
	s.worldLock.Lock()
	defer s.worldLock.Unlock()
	err := json.Unmarshal(state, s)

	if err != nil {
		return err
	}

	s.Root.recoverAndclean()
	return nil
}

func (s *Store) JsonStats() []byte {
	s.Stats.Watchers = uint64(s.WatcherHub.count)
	return s.Stats.toJson()
}
<commit_msg>minor clean up<commit_after>package store

import (
	"encoding/json"
	"fmt"
	"path"
	"sort"
	"strings"
	"sync"
	"time"

	etcdErr "github.com/coreos/etcd/error"
)

type Store struct {
	Root       *Node
	WatcherHub *watcherHub
	Index      uint64
	Term       uint64
	Stats      *Stats
	worldLock  sync.RWMutex // stop the world lock. Used to do snapshot
}

func New() *Store {
	s := new(Store)
	s.Root = newDir("/", 0, 0, nil, "", Permanent)
	s.Stats = newStats()
	s.WatcherHub = newWatchHub(1000)

	return s
}

func (s *Store) Get(nodePath string, recursive, sorted bool, index uint64, term uint64) (*Event, error) {
	s.worldLock.RLock()
	defer s.worldLock.RUnlock()

	nodePath = path.Clean(path.Join("/", nodePath))

	n, err := s.internalGet(nodePath, index, term)

	if err != nil {
		s.Stats.Inc(GetFail)
		return nil, err
	}

	e := newEvent(Get, nodePath, index, term)

	if n.IsDir() { // node is dir
		e.Dir = true

		children, _ := n.List()
		e.KVPairs = make([]KeyValuePair, len(children))

		// we do not use the index in the children slice directly
		// we need to skip the hidden one
		i := 0

		for _, child := range children {

			if child.IsHidden() { // get will not list hidden node
				continue
			}

			e.KVPairs[i] = child.Pair(recursive, sorted)

			i++
		}

		// eliminate hidden nodes
		e.KVPairs = e.KVPairs[:i]

		rootPairs := KeyValuePair{
			KVPairs: e.KVPairs,
		}

		if sorted {
			sort.Sort(rootPairs)
		}

	} else { // node is file
		e.Value = n.Value
	}

	if n.ExpireTime.Sub(Permanent) != 0 {
		e.Expiration = &n.ExpireTime
		e.TTL = int64(n.ExpireTime.Sub(time.Now())/time.Second) + 1
	}

	s.Stats.Inc(GetSuccess)

	return e, nil
}

// Create function creates the Node at nodePath. Create will help to create intermediate directories with no ttl.
// If the node has already existed, create will fail.
// If any node on the path is a file, create will fail.
func (s *Store) Create(nodePath string, value string, expireTime time.Time, index uint64, term uint64) (*Event, error) {
	s.worldLock.RLock()
	defer s.worldLock.RUnlock()

	nodePath = path.Clean(path.Join("/", nodePath))

	// make sure we can create the node
	_, err := s.internalGet(nodePath, index, term)

	if err == nil { // key already exists
		s.Stats.Inc(SetFail)
		return nil, etcdErr.NewError(etcdErr.EcodeNodeExist, nodePath)
	}

	etcdError, _ := err.(etcdErr.Error)

	if etcdError.ErrorCode == 104 { // we cannot create the key due to meet a file while walking
		s.Stats.Inc(SetFail)
		return nil, err
	}

	dir, _ := path.Split(nodePath)

	// walk through the nodePath, create dirs and get the last directory node
	d, err := s.walk(dir, s.checkDir)

	if err != nil {
		s.Stats.Inc(SetFail)
		return nil, err
	}

	e := newEvent(Create, nodePath, s.Index, s.Term)

	var n *Node

	if len(value) != 0 { // create file
		e.Value = value

		n = newFile(nodePath, value, s.Index, s.Term, d, "", expireTime)

	} else { // create directory
		e.Dir = true

		n = newDir(nodePath, s.Index, s.Term, d, "", expireTime)

	}

	err = d.Add(n)

	if err != nil {
		s.Stats.Inc(SetFail)
		return nil, err
	}

	// Node with TTL
	if expireTime.Sub(Permanent) != 0 {
		n.Expire()
		e.Expiration = &n.ExpireTime
		e.TTL = int64(expireTime.Sub(time.Now())/time.Second) + 1
	}

	s.WatcherHub.notify(e)
	s.Stats.Inc(SetSuccess)
	return e, nil
}

// Update function updates the value/ttl of the node.
// If the node is a file, the value and the ttl can be updated.
// If the node is a directory, only the ttl can be updated.
func (s *Store) Update(nodePath string, value string, expireTime time.Time, index uint64, term uint64) (*Event, error) {
	s.worldLock.RLock()
	defer s.worldLock.RUnlock()

	n, err := s.internalGet(nodePath, index, term)

	if err != nil { // if the node does not exist, return error
		s.Stats.Inc(UpdateFail)
		return nil, err
	}

	e := newEvent(Update, nodePath, s.Index, s.Term)

	if n.IsDir() { // if the node is a directory, we can only update ttl

		if len(value) != 0 {
			s.Stats.Inc(UpdateFail)
			return nil, etcdErr.NewError(etcdErr.EcodeNotFile, nodePath)
		}

	} else { // if the node is a file, we can update value and ttl
		e.PrevValue = n.Value

		if len(value) != 0 {
			e.Value = value
		}

		n.Write(value, index, term)
	}

	// update ttl
	if !n.IsPermanent() {
		n.stopExpire <- true
	}

	if expireTime.Sub(Permanent) != 0 {
		n.ExpireTime = expireTime
		n.Expire()
		e.Expiration = &n.ExpireTime
		e.TTL = int64(expireTime.Sub(time.Now())/time.Second) + 1
	}

	s.WatcherHub.notify(e)
	s.Stats.Inc(UpdateSuccess)
	return e, nil
}

func (s *Store) TestAndSet(nodePath string, prevValue string, prevIndex uint64,
	value string, expireTime time.Time, index uint64, term uint64) (*Event, error) {

	s.worldLock.RLock()
	defer s.worldLock.RUnlock()

	f, err := s.internalGet(nodePath, index, term)

	if err != nil {
		s.Stats.Inc(TestAndSetFail)
		return nil, err
	}

	if f.IsDir() { // can only test and set file
		s.Stats.Inc(TestAndSetFail)
		return nil, etcdErr.NewError(etcdErr.EcodeNotFile, nodePath)
	}

	if f.Value == prevValue || f.ModifiedIndex == prevIndex {
		// if test succeed, write the value
		e := newEvent(TestAndSet, nodePath, index, term)
		e.PrevValue = f.Value
		e.Value = value
		f.Write(value, index, term)

		s.WatcherHub.notify(e)
		s.Stats.Inc(TestAndSetSuccess)
		return e, nil
	}

	cause := fmt.Sprintf("[%v != %v] [%v != %v]", prevValue, f.Value, prevIndex, f.ModifiedIndex)
	s.Stats.Inc(TestAndSetFail)
	return nil, etcdErr.NewError(etcdErr.EcodeTestFailed, cause)
}

// Delete function deletes the node at the given path.
// If the node is a directory, recursive must be true to delete it.
func (s *Store) Delete(nodePath string, recursive bool, index uint64, term uint64) (*Event, error) {
	s.worldLock.RLock()
	defer s.worldLock.RUnlock()

	n, err := s.internalGet(nodePath, index, term)

	if err != nil { // if the node does not exist, return error
		s.Stats.Inc(DeleteFail)
		return nil, err
	}

	e := newEvent(Delete, nodePath, index, term)

	if n.IsDir() {
		e.Dir = true
	} else {
		e.PrevValue = n.Value
	}

	callback := func(path string) { // notify function
		s.WatcherHub.notifyWithPath(e, path, true)
	}

	err = n.Remove(recursive, callback)

	if err != nil {
		s.Stats.Inc(DeleteFail)
		return nil, err
	}

	s.WatcherHub.notify(e)
	s.Stats.Inc(DeleteSuccess)

	return e, nil
}

func (s *Store) Watch(prefix string, recursive bool, sinceIndex uint64, index uint64, term uint64) (<-chan *Event, error) {
	s.worldLock.RLock()
	defer s.worldLock.RUnlock()

	s.Index, s.Term = index, term

	if sinceIndex == 0 {
		return s.WatcherHub.watch(prefix, recursive, index+1)
	}

	return s.WatcherHub.watch(prefix, recursive, sinceIndex)
}

// walk function walks all the nodePath and apply the walkFunc on each directory
func (s *Store) walk(nodePath string, walkFunc func(prev *Node, component string) (*Node, error)) (*Node, error) {
	components := strings.Split(nodePath, "/")

	curr := s.Root

	var err error
	for i := 1; i < len(components); i++ {
		if len(components[i]) == 0 { // ignore empty string
			return curr, nil
		}

		curr, err = walkFunc(curr, components[i])
		if err != nil {
			return nil, err
		}

	}

	return curr, nil
}

// InternalGet function get the node of the given nodePath.
func (s *Store) internalGet(nodePath string, index uint64, term uint64) (*Node, error) {
	nodePath = path.Clean(path.Join("/", nodePath))

	// update file system known index and term
	s.Index, s.Term = index, term

	walkFunc := func(parent *Node, name string) (*Node, error) {

		if !parent.IsDir() {
			return nil, etcdErr.NewError(etcdErr.EcodeNotDir, parent.Path)
		}

		child, ok := parent.Children[name]
		if ok {
			return child, nil
		}

		return nil, etcdErr.NewError(etcdErr.EcodeKeyNotFound, path.Join(parent.Path, name))
	}

	f, err := s.walk(nodePath, walkFunc)

	if err != nil {
		return nil, err
	}

	return f, nil
}

// checkDir function will check whether the component is a directory under parent node.
// If it is a directory, this function will return the pointer to that node.
// If it does not exist, this function will create a new directory and return the pointer to that node.
// If it is a file, this function will return error.
func (s *Store) checkDir(parent *Node, dirName string) (*Node, error) {
	subDir, ok := parent.Children[dirName]

	if ok {
		return subDir, nil
	}

	n := newDir(path.Join(parent.Path, dirName), s.Index, s.Term, parent, parent.ACL, Permanent)

	parent.Children[dirName] = n

	return n, nil
}

// Save function saves the static state of the store system.
// Save function will not be able to save the state of watchers.
// Save function will not save the parent field of the node. Or there will
// be cyclic dependencies issue for the json package.
func (s *Store) Save() ([]byte, error) {
	s.worldLock.Lock()

	clonedStore := New()
	clonedStore.Index = s.Index
	clonedStore.Term = s.Term
	clonedStore.Root = s.Root.Clone()
	clonedStore.WatcherHub = s.WatcherHub.clone()
	clonedStore.Stats = s.Stats.clone()

	s.worldLock.Unlock()

	b, err := json.Marshal(clonedStore)

	if err != nil {
		return nil, err
	}

	return b, nil
}

// recovery function recovery the store system from a static state.
// It needs to recovery the parent field of the nodes.
// It needs to delete the expired nodes since the saved time and also
// need to create monitor go routines.
func (s *Store) Recovery(state []byte) error {
	s.worldLock.Lock()
	defer s.worldLock.Unlock()
	err := json.Unmarshal(state, s)

	if err != nil {
		return err
	}

	s.Root.recoverAndclean()
	return nil
}

func (s *Store) JsonStats() []byte {
	s.Stats.Watchers = uint64(s.WatcherHub.count)
	return s.Stats.toJson()
}
<|endoftext|>


<commit_before>package store

import (
	"encoding/json"
	"fmt"
	"path"
	"strconv"
	"time"
)

//------------------------------------------------------------------------------
//
// Typedefs
//
//------------------------------------------------------------------------------

// The main struct of the Key-Value store
type Store struct {

	// key-value store structure
	Tree *tree

	// WatcherHub is where we register all the clients
	// who issue a watch request
	watcher *WatcherHub

	// The string channel to send messages to the outside world
	// Now we use it to send changes to the hub of the web service
	messager *chan string

	// A map to keep the recent response to the clients
	ResponseMap map[string]Response

	// The max number of the recent responses we can record
	ResponseMaxSize int

	// The current number of the recent responses we have recorded
	ResponseCurrSize uint

	// The index of the first recent responses we have
	ResponseStartIndex uint64

	// Current index of the raft machine
	Index uint64

	// Basic statistics information of etcd storage
	BasicStats EtcdStats
}

// A Node represents a Value in the Key-Value pair in the store
// It has its value, expire time and a channel used to update the
// expire time (since we do countdown in a go routine, we need to
// communicate with it via channel)
type Node struct {
	// The string value of the node
	Value string `json:"value"`

	// If the node is a permanent one the ExprieTime will be Unix(0,0)
	// Otherwise after the expireTime, the node will be deleted
	ExpireTime time.Time `json:"expireTime"`

	// A channel to update the expireTime of the node
	update chan time.Time `json:"-"`
}

// The response from the store to the user who issue a command
type Response struct {
	Action    string `json:"action"`
	Key       string `json:"key"`
	Dir       bool   `json:"dir,omitempty"`
	PrevValue string `json:"prevValue,omitempty"`
	Value     string `json:"value,omitempty"`

	// If the key did not exist before the action,
	// this field should be set to true
	NewKey bool `json:"newKey,omitempty"`

	Expiration *time.Time `json:"expiration,omitempty"`

	// Time to live in second
	TTL int64 `json:"ttl,omitempty"`

	// The command index of the raft machine when the command is executed
	Index uint64 `json:"index"`
}

// A listNode represent the simplest Key-Value pair with its type
// It is only used when do list opeartion
// We want to have a file system like store, thus we distingush "file"
// and "directory"
type ListNode struct {
	Key   string
	Value string
	Type  string
}

var PERMANENT = time.Unix(0, 0)

//------------------------------------------------------------------------------
//
// Methods
//
//------------------------------------------------------------------------------

// Create a new stroe
// Arguement max is the max number of response we want to record
func CreateStore(max int) *Store {
	s := new(Store)

	s.messager = nil

	s.ResponseMap = make(map[string]Response)
	s.ResponseStartIndex = 0
	s.ResponseMaxSize = max
	s.ResponseCurrSize = 0

	s.Tree = &tree{
		&treeNode{
			Node{
				"/",
				time.Unix(0, 0),
				nil,
			},
			true,
			make(map[string]*treeNode),
		},
	}

	s.watcher = createWatcherHub()

	return s
}

// Set the messager of the store
func (s *Store) SetMessager(messager *chan string) {
	s.messager = messager
}

// Set the key to value with expiration time
func (s *Store) Set(key string, value string, expireTime time.Time, index uint64) ([]byte, error) {

	//Update index
	s.Index = index

	//Update stats
	s.BasicStats.Sets++

	key = path.Clean("/" + key)

	isExpire := !expireTime.Equal(PERMANENT)

	// base response
	resp := Response{
		Action: "SET",
		Key:    key,
		Value:  value,
		Index:  index,
	}

	// When the slow follower receive the set command
	// the key may be expired, we should not add the node
	// also if the node exist, we need to delete the node
	if isExpire && expireTime.Sub(time.Now()) < 0 {
		return s.Delete(key, index)
	}

	var TTL int64

	// Update ttl
	if isExpire {
		TTL = int64(expireTime.Sub(time.Now()) / time.Second)
		resp.Expiration = &expireTime
		resp.TTL = TTL
	}

	// Get the node
	node, ok := s.Tree.get(key)

	if ok {
		// Update when node exists

		// Node is not permanent
		if !node.ExpireTime.Equal(PERMANENT) {

			// If node is not permanent
			// Update its expireTime
			node.update <- expireTime

		} else {

			// If we want the permanent node to have expire time
			// We need to create create a go routine with a channel
			if isExpire {
				node.update = make(chan time.Time)
				go s.monitorExpiration(key, node.update, expireTime)
			}

		}

		// Update the information of the node
		s.Tree.set(key, Node{value, expireTime, node.update})

		resp.PrevValue = node.Value

		s.watcher.notify(resp)

		msg, err := json.Marshal(resp)

		// Send to the messager
		if s.messager != nil && err == nil {

			*s.messager <- string(msg)
		}

		s.addToResponseMap(index, &resp)

		return msg, err

		// Add new node
	} else {

		update := make(chan time.Time)

		ok := s.Tree.set(key, Node{value, expireTime, update})

		if !ok {
			err := NotFile(key)
			return nil, err
		}

		if isExpire {
			go s.monitorExpiration(key, update, expireTime)
		}

		resp.NewKey = true

		msg, err := json.Marshal(resp)

		// Nofity the watcher
		s.watcher.notify(resp)

		// Send to the messager
		if s.messager != nil && err == nil {

			*s.messager <- string(msg)
		}

		s.addToResponseMap(index, &resp)
		return msg, err
	}

}

// Get the value of the key and return the raw response
func (s *Store) internalGet(key string) *Response {

	key = path.Clean("/" + key)

	node, ok := s.Tree.get(key)

	if ok {
		var TTL int64
		var isExpire bool = false

		isExpire = !node.ExpireTime.Equal(PERMANENT)

		resp := &Response{
			Action: "GET",
			Key:    key,
			Value:  node.Value,
			Index:  s.Index,
		}

		// Update ttl
		if isExpire {
			TTL = int64(node.ExpireTime.Sub(time.Now()) / time.Second)
			resp.Expiration = &node.ExpireTime
			resp.TTL = TTL
		}

		return resp

	} else {
		// we do not found the key
		return nil
	}
}

// Get all the items under key
// If key is a file return the file
// If key is a directory reuturn an array of files
func (s *Store) Get(key string) ([]byte, error) {
	resps, err := s.RawGet(key)

	if err != nil {
		return nil, err
	}

	if len(resps) == 1 {
		return json.Marshal(resps[0])
	}

	return json.Marshal(resps)
}

func (s *Store) RawGet(key string) ([]*Response, error) {
	//Update stats
	s.BasicStats.Gets++

	key = path.Clean("/" + key)

	nodes, keys, dirs, ok := s.Tree.list(key)

	if ok {
		resps := make([]*Response, len(nodes))
		for i := 0; i < len(nodes); i++ {

			var TTL int64
			var isExpire bool = false

			isExpire = !nodes[i].ExpireTime.Equal(PERMANENT)

			resps[i] = &Response{
				Action: "GET",
				Index:  s.Index,
				Key:    path.Join(key, keys[i]),
			}

			if !dirs[i] {
				resps[i].Value = nodes[i].Value
			} else {
				resps[i].Dir = true
			}

			// Update ttl
			if isExpire {
				TTL = int64(nodes[i].ExpireTime.Sub(time.Now()) / time.Second)
				resps[i].Expiration = &nodes[i].ExpireTime
				resps[i].TTL = TTL
			}

		}

		return resps, nil
	}

	err := NotFoundError(key)
	return nil, err
}

// Delete the key
func (s *Store) Delete(key string, index uint64) ([]byte, error) {

	//Update stats
	s.BasicStats.Deletes++

	key = path.Clean("/" + key)

	//Update index
	s.Index = index

	node, ok := s.Tree.get(key)

	if ok {

		resp := Response{
			Action:    "DELETE",
			Key:       key,
			PrevValue: node.Value,
			Index:     index,
		}

		if node.ExpireTime.Equal(PERMANENT) {

			s.Tree.delete(key)

		} else {
			resp.Expiration = &node.ExpireTime
			// Kill the expire go routine
			node.update <- PERMANENT
			s.Tree.delete(key)

		}

		msg, err := json.Marshal(resp)

		s.watcher.notify(resp)

		// notify the messager
		if s.messager != nil && err == nil {

			*s.messager <- string(msg)
		}

		s.addToResponseMap(index, &resp)

		return msg, err

	} else {
		err := NotFoundError(key)
		return nil, err
	}
}

// Set the value of the key to the value if the given prevValue is equal to the value of the key
func (s *Store) TestAndSet(key string, prevValue string, value string, expireTime time.Time, index uint64) ([]byte, error) {
	//Update stats
	s.BasicStats.TestAndSets++

	resp := s.internalGet(key)

	if resp == nil {
		err := NotFoundError(key)
		return nil, err
	}

	if resp.Value == prevValue {

		// If test success, do set
		return s.Set(key, value, expireTime, index)
	} else {

		// If fails, return err
		err := TestFail(fmt.Sprintf("TestAndSet: %s!=%s", resp.Value, prevValue))
		return nil, err
	}

}

// Add a channel to the watchHub.
// The watchHub will send response to the channel when any key under the prefix
// changes [since the sinceIndex if given]
func (s *Store) AddWatcher(prefix string, watcher *Watcher, sinceIndex uint64) error {
	return s.watcher.addWatcher(prefix, watcher, sinceIndex, s.ResponseStartIndex, s.Index, &s.ResponseMap)
}

// This function should be created as a go routine to delete the key-value pair
// when it reaches expiration time

func (s *Store) monitorExpiration(key string, update chan time.Time, expireTime time.Time) {

	duration := expireTime.Sub(time.Now())

	for {
		select {

		// Timeout delete the node
		case <-time.After(duration):
			node, ok := s.Tree.get(key)

			if !ok {
				return

			} else {

				s.Tree.delete(key)

				resp := Response{
					Action:     "DELETE",
					Key:        key,
					PrevValue:  node.Value,
					Expiration: &node.ExpireTime,
					Index:      s.Index,
				}

				msg, err := json.Marshal(resp)

				s.watcher.notify(resp)

				// notify the messager
				if s.messager != nil && err == nil {

					*s.messager <- string(msg)
				}

				return

			}

		case updateTime := <-update:
			// Update duration
			// If the node become a permanent one, the go routine is
			// not needed
			if updateTime.Equal(PERMANENT) {
				return
			}

			// Update duration
			duration = updateTime.Sub(time.Now())
		}
	}
}

// When we receive a command that will change the state of the key-value store
// We will add the result of it to the ResponseMap for the use of watch command
// Also we may remove the oldest response when we add new one
func (s *Store) addToResponseMap(index uint64, resp *Response) {

	// zero case
	if s.ResponseMaxSize == 0 {
		return
	}

	strIndex := strconv.FormatUint(index, 10)
	s.ResponseMap[strIndex] = *resp

	// unlimited
	if s.ResponseMaxSize < 0 {
		s.ResponseCurrSize++
		return
	}

	// if we reach the max point, we need to delete the most latest
	// response and update the startIndex
	if s.ResponseCurrSize == uint(s.ResponseMaxSize) {
		s.ResponseStartIndex++
		delete(s.ResponseMap, strconv.FormatUint(s.ResponseStartIndex, 10))
	} else {
		s.ResponseCurrSize++
	}
}

// Save the current state of the storage system
func (s *Store) Save() ([]byte, error) {
	b, err := json.Marshal(s)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}
	return b, nil
}

// Recovery the state of the stroage system from a previous state
func (s *Store) Recovery(state []byte) error {
	err := json.Unmarshal(state, s)

	// The only thing need to change after the recovery is the
	// node with expiration time, we need to delete all the node
	// that have been expired and setup go routines to monitor the
	// other ones
	s.checkExpiration()

	return err
}

// Clean the expired nodes
// Set up go routines to mon
func (s *Store) checkExpiration() {
	s.Tree.traverse(s.checkNode, false)
}

// Check each node
func (s *Store) checkNode(key string, node *Node) {

	if node.ExpireTime.Equal(PERMANENT) {
		return
	} else {
		if node.ExpireTime.Sub(time.Now()) >= time.Second {

			node.update = make(chan time.Time)
			go s.monitorExpiration(key, node.update, node.ExpireTime)

		} else {
			// we should delete this node
			s.Tree.delete(key)
		}
	}
}
<commit_msg>add space in comments<commit_after>package store

import (
	"encoding/json"
	"fmt"
	"path"
	"strconv"
	"time"
)

//------------------------------------------------------------------------------
//
// Typedefs
//
//------------------------------------------------------------------------------

// The main struct of the Key-Value store
type Store struct {

	// key-value store structure
	Tree *tree

	// WatcherHub is where we register all the clients
	// who issue a watch request
	watcher *WatcherHub

	// The string channel to send messages to the outside world
	// Now we use it to send changes to the hub of the web service
	messager *chan string

	// A map to keep the recent response to the clients
	ResponseMap map[string]Response

	// The max number of the recent responses we can record
	ResponseMaxSize int

	// The current number of the recent responses we have recorded
	ResponseCurrSize uint

	// The index of the first recent responses we have
	ResponseStartIndex uint64

	// Current index of the raft machine
	Index uint64

	// Basic statistics information of etcd storage
	BasicStats EtcdStats
}

// A Node represents a Value in the Key-Value pair in the store
// It has its value, expire time and a channel used to update the
// expire time (since we do countdown in a go routine, we need to
// communicate with it via channel)
type Node struct {
	// The string value of the node
	Value string `json:"value"`

	// If the node is a permanent one the ExprieTime will be Unix(0,0)
	// Otherwise after the expireTime, the node will be deleted
	ExpireTime time.Time `json:"expireTime"`

	// A channel to update the expireTime of the node
	update chan time.Time `json:"-"`
}

// The response from the store to the user who issue a command
type Response struct {
	Action    string `json:"action"`
	Key       string `json:"key"`
	Dir       bool   `json:"dir,omitempty"`
	PrevValue string `json:"prevValue,omitempty"`
	Value     string `json:"value,omitempty"`

	// If the key did not exist before the action,
	// this field should be set to true
	NewKey bool `json:"newKey,omitempty"`

	Expiration *time.Time `json:"expiration,omitempty"`

	// Time to live in second
	TTL int64 `json:"ttl,omitempty"`

	// The command index of the raft machine when the command is executed
	Index uint64 `json:"index"`
}

// A listNode represent the simplest Key-Value pair with its type
// It is only used when do list opeartion
// We want to have a file system like store, thus we distingush "file"
// and "directory"
type ListNode struct {
	Key   string
	Value string
	Type  string
}

var PERMANENT = time.Unix(0, 0)

//------------------------------------------------------------------------------
//
// Methods
//
//------------------------------------------------------------------------------

// Create a new stroe
// Arguement max is the max number of response we want to record
func CreateStore(max int) *Store {
	s := new(Store)

	s.messager = nil

	s.ResponseMap = make(map[string]Response)
	s.ResponseStartIndex = 0
	s.ResponseMaxSize = max
	s.ResponseCurrSize = 0

	s.Tree = &tree{
		&treeNode{
			Node{
				"/",
				time.Unix(0, 0),
				nil,
			},
			true,
			make(map[string]*treeNode),
		},
	}

	s.watcher = createWatcherHub()

	return s
}

// Set the messager of the store
func (s *Store) SetMessager(messager *chan string) {
	s.messager = messager
}

// Set the key to value with expiration time
func (s *Store) Set(key string, value string, expireTime time.Time, index uint64) ([]byte, error) {

	//Update index
	s.Index = index

	//Update stats
	s.BasicStats.Sets++

	key = path.Clean("/" + key)

	isExpire := !expireTime.Equal(PERMANENT)

	// base response
	resp := Response{
		Action: "SET",
		Key:    key,
		Value:  value,
		Index:  index,
	}

	// When the slow follower receive the set command
	// the key may be expired, we should not add the node
	// also if the node exist, we need to delete the node
	if isExpire && expireTime.Sub(time.Now()) < 0 {
		return s.Delete(key, index)
	}

	var TTL int64

	// Update ttl
	if isExpire {
		TTL = int64(expireTime.Sub(time.Now()) / time.Second)
		resp.Expiration = &expireTime
		resp.TTL = TTL
	}

	// Get the node
	node, ok := s.Tree.get(key)

	if ok {
		// Update when node exists

		// Node is not permanent
		if !node.ExpireTime.Equal(PERMANENT) {

			// If node is not permanent
			// Update its expireTime
			node.update <- expireTime

		} else {

			// If we want the permanent node to have expire time
			// We need to create create a go routine with a channel
			if isExpire {
				node.update = make(chan time.Time)
				go s.monitorExpiration(key, node.update, expireTime)
			}

		}

		// Update the information of the node
		s.Tree.set(key, Node{value, expireTime, node.update})

		resp.PrevValue = node.Value

		s.watcher.notify(resp)

		msg, err := json.Marshal(resp)

		// Send to the messager
		if s.messager != nil && err == nil {

			*s.messager <- string(msg)
		}

		s.addToResponseMap(index, &resp)

		return msg, err

		// Add new node
	} else {

		update := make(chan time.Time)

		ok := s.Tree.set(key, Node{value, expireTime, update})

		if !ok {
			err := NotFile(key)
			return nil, err
		}

		if isExpire {
			go s.monitorExpiration(key, update, expireTime)
		}

		resp.NewKey = true

		msg, err := json.Marshal(resp)

		// Nofity the watcher
		s.watcher.notify(resp)

		// Send to the messager
		if s.messager != nil && err == nil {

			*s.messager <- string(msg)
		}

		s.addToResponseMap(index, &resp)
		return msg, err
	}

}

// Get the value of the key and return the raw response
func (s *Store) internalGet(key string) *Response {

	key = path.Clean("/" + key)

	node, ok := s.Tree.get(key)

	if ok {
		var TTL int64
		var isExpire bool = false

		isExpire = !node.ExpireTime.Equal(PERMANENT)

		resp := &Response{
			Action: "GET",
			Key:    key,
			Value:  node.Value,
			Index:  s.Index,
		}

		// Update ttl
		if isExpire {
			TTL = int64(node.ExpireTime.Sub(time.Now()) / time.Second)
			resp.Expiration = &node.ExpireTime
			resp.TTL = TTL
		}

		return resp

	} else {
		// we do not found the key
		return nil
	}
}

// Get all the items under key
// If key is a file return the file
// If key is a directory reuturn an array of files
func (s *Store) Get(key string) ([]byte, error) {
	resps, err := s.RawGet(key)

	if err != nil {
		return nil, err
	}

	if len(resps) == 1 {
		return json.Marshal(resps[0])
	}

	return json.Marshal(resps)
}

func (s *Store) RawGet(key string) ([]*Response, error) {
	// Update stats
	s.BasicStats.Gets++

	key = path.Clean("/" + key)

	nodes, keys, dirs, ok := s.Tree.list(key)

	if ok {
		resps := make([]*Response, len(nodes))
		for i := 0; i < len(nodes); i++ {

			var TTL int64
			var isExpire bool = false

			isExpire = !nodes[i].ExpireTime.Equal(PERMANENT)

			resps[i] = &Response{
				Action: "GET",
				Index:  s.Index,
				Key:    path.Join(key, keys[i]),
			}

			if !dirs[i] {
				resps[i].Value = nodes[i].Value
			} else {
				resps[i].Dir = true
			}

			// Update ttl
			if isExpire {
				TTL = int64(nodes[i].ExpireTime.Sub(time.Now()) / time.Second)
				resps[i].Expiration = &nodes[i].ExpireTime
				resps[i].TTL = TTL
			}

		}

		return resps, nil
	}

	err := NotFoundError(key)
	return nil, err
}

// Delete the key
func (s *Store) Delete(key string, index uint64) ([]byte, error) {

	// Update stats
	s.BasicStats.Deletes++

	key = path.Clean("/" + key)

	// Update index
	s.Index = index

	node, ok := s.Tree.get(key)

	if ok {

		resp := Response{
			Action:    "DELETE",
			Key:       key,
			PrevValue: node.Value,
			Index:     index,
		}

		if node.ExpireTime.Equal(PERMANENT) {

			s.Tree.delete(key)

		} else {
			resp.Expiration = &node.ExpireTime
			// Kill the expire go routine
			node.update <- PERMANENT
			s.Tree.delete(key)

		}

		msg, err := json.Marshal(resp)

		s.watcher.notify(resp)

		// notify the messager
		if s.messager != nil && err == nil {

			*s.messager <- string(msg)
		}

		s.addToResponseMap(index, &resp)

		return msg, err

	} else {
		err := NotFoundError(key)
		return nil, err
	}
}

// Set the value of the key to the value if the given prevValue is equal to the value of the key
func (s *Store) TestAndSet(key string, prevValue string, value string, expireTime time.Time, index uint64) ([]byte, error) {
	// Update stats
	s.BasicStats.TestAndSets++

	resp := s.internalGet(key)

	if resp == nil {
		err := NotFoundError(key)
		return nil, err
	}

	if resp.Value == prevValue {

		// If test success, do set
		return s.Set(key, value, expireTime, index)
	} else {

		// If fails, return err
		err := TestFail(fmt.Sprintf("TestAndSet: %s!=%s", resp.Value, prevValue))
		return nil, err
	}

}

// Add a channel to the watchHub.
// The watchHub will send response to the channel when any key under the prefix
// changes [since the sinceIndex if given]
func (s *Store) AddWatcher(prefix string, watcher *Watcher, sinceIndex uint64) error {
	return s.watcher.addWatcher(prefix, watcher, sinceIndex, s.ResponseStartIndex, s.Index, &s.ResponseMap)
}

// This function should be created as a go routine to delete the key-value pair
// when it reaches expiration time

func (s *Store) monitorExpiration(key string, update chan time.Time, expireTime time.Time) {

	duration := expireTime.Sub(time.Now())

	for {
		select {

		// Timeout delete the node
		case <-time.After(duration):
			node, ok := s.Tree.get(key)

			if !ok {
				return

			} else {

				s.Tree.delete(key)

				resp := Response{
					Action:     "DELETE",
					Key:        key,
					PrevValue:  node.Value,
					Expiration: &node.ExpireTime,
					Index:      s.Index,
				}

				msg, err := json.Marshal(resp)

				s.watcher.notify(resp)

				// notify the messager
				if s.messager != nil && err == nil {

					*s.messager <- string(msg)
				}

				return

			}

		case updateTime := <-update:
			// Update duration
			// If the node become a permanent one, the go routine is
			// not needed
			if updateTime.Equal(PERMANENT) {
				return
			}

			// Update duration
			duration = updateTime.Sub(time.Now())
		}
	}
}

// When we receive a command that will change the state of the key-value store
// We will add the result of it to the ResponseMap for the use of watch command
// Also we may remove the oldest response when we add new one
func (s *Store) addToResponseMap(index uint64, resp *Response) {

	// zero case
	if s.ResponseMaxSize == 0 {
		return
	}

	strIndex := strconv.FormatUint(index, 10)
	s.ResponseMap[strIndex] = *resp

	// unlimited
	if s.ResponseMaxSize < 0 {
		s.ResponseCurrSize++
		return
	}

	// if we reach the max point, we need to delete the most latest
	// response and update the startIndex
	if s.ResponseCurrSize == uint(s.ResponseMaxSize) {
		s.ResponseStartIndex++
		delete(s.ResponseMap, strconv.FormatUint(s.ResponseStartIndex, 10))
	} else {
		s.ResponseCurrSize++
	}
}

// Save the current state of the storage system
func (s *Store) Save() ([]byte, error) {
	b, err := json.Marshal(s)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}
	return b, nil
}

// Recovery the state of the stroage system from a previous state
func (s *Store) Recovery(state []byte) error {
	err := json.Unmarshal(state, s)

	// The only thing need to change after the recovery is the
	// node with expiration time, we need to delete all the node
	// that have been expired and setup go routines to monitor the
	// other ones
	s.checkExpiration()

	return err
}

// Clean the expired nodes
// Set up go routines to mon
func (s *Store) checkExpiration() {
	s.Tree.traverse(s.checkNode, false)
}

// Check each node
func (s *Store) checkNode(key string, node *Node) {

	if node.ExpireTime.Equal(PERMANENT) {
		return
	} else {
		if node.ExpireTime.Sub(time.Now()) >= time.Second {

			node.update = make(chan time.Time)
			go s.monitorExpiration(key, node.update, node.ExpireTime)

		} else {
			// we should delete this node
			s.Tree.delete(key)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"database/sql"
	"fmt"
	"github.com/kardianos/osext"
	_ "github.com/mattn/go-sqlite3"
	"github.com/spf13/viper"
	"html/template"
	"net"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"time"
)

type DbContext struct {
	Db     *sql.DB
	Insert *sql.Stmt
}

func (c *DbContext) Close() {
	if c.Insert != nil {
		c.Insert.Close()
	}
	if c.Db != nil {
		c.Db.Close()
	}
}

func loadConfig() {
	viper.SetConfigName("pbxlog")
	viper.SetConfigType("yaml")

	viper.AddConfigPath("$HOME")
	viper.AddConfigPath("$HOME/.pbxlog")
	viper.AddConfigPath(".")

	viper.SetDefault("pabx", "")
	viper.SetDefault("webui", "")
	viper.SetDefault("dump-file", "")
	viper.SetDefault("error-file", "")
	viper.SetDefault("calls-db", "")

	err := viper.ReadInConfig()
	if err != nil {
		panic(fmt.Errorf("Fatal error config file: %s \n", err))
	}

	fmt.Printf("Using pabx = %s\n", viper.GetString("pabx"))
	fmt.Printf("Using webui = %s\n", viper.GetString("webui"))
	fmt.Printf("Using dump-file = %s\n", viper.GetString("dump-file"))
	fmt.Printf("Using error-file = %s\n", viper.GetString("error-file"))
	fmt.Printf("Using calls-db = %s\n", viper.GetString("calls-db"))

	if viper.GetString("pabx") == "" || viper.GetString("calls-db") == "" {
		panic("pabx and calls-db configuration values required")
	}
}

func connectToPABX() net.Conn {
	conn, err := net.Dial("tcp", viper.GetString("pabx"))
	panicErr(err)
	return conn
}

func openDumpFile(configItem string) *os.File {
	filename := viper.GetString(configItem)
	if filename == "" {
		return nil
	}

	dumpfile, err := os.OpenFile(filename,
		os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
	panicErr(err)

	return dumpfile
}

type CDR struct {
	Callid    int
	Extension int
	ExtName   string
	Auth      string
	AuthName  string
	Calltime  string
	Duration  string
	Code      string
	Dialed    string
	Account   string
	Cost      string
	Clid      string
	Clidname  string
	Gpno      string
	Ringtime  string
}

func openCallsDatabase() *DbContext {
	ctx := new(DbContext)

	var err error
	ctx.Db, err = sql.Open("sqlite3", viper.GetString("calls-db"))
	panicErr(err)

	_, err = ctx.Db.Exec("CREATE TABLE IF NOT EXISTS calls (" +
		"callid INTEGER, extension INTEGER, auth TEXT, calltime TEXT, duration TEXT, " +
		"code TEXT, dialed TEXT, account TEXT, cost REAL, clid TEXT, " +
		"clidname TEXT, gpno TEXT, ringtime TEXT);")
	panicErr(err)

	_, err = ctx.Db.Exec("CREATE TABLE IF NOT EXISTS extensions (num INTEGER, name TEXT);")
	panicErr(err)

	ctx.Insert, err = ctx.Db.Prepare("INSERT INTO calls(callid, extension, auth, calltime, " +
		"duration, code, dialed, account, cost, clid, clidname, gpno, ringtime) " +
		"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
	panicErr(err)

	return ctx
}

func insertCDR(ctx *DbContext, cdr *CDR) {
	_, err := ctx.Insert.Exec(cdr.Callid, cdr.Extension, cdr.Auth,
		cdr.Calltime, cdr.Duration, cdr.Code, cdr.Dialed, cdr.Account,
		cdr.Cost, cdr.Clid, cdr.Clidname, cdr.Gpno, cdr.Ringtime)
	panicErr(err)
}

func splitData(str string) *CDR {
	n := len(str)
	if n != 154 && n != 122 {
		return nil
	}
	t := time.Now()

	var cdr CDR
	cdr.Callid, _ = strconv.Atoi(strings.TrimSpace(str[2:8]))
	cdr.Extension, _ = strconv.Atoi(strings.TrimSpace(str[9:15]))
	cdr.Auth = strings.TrimSpace(str[16:25])
	cdr.Calltime = strings.TrimSpace(fmt.Sprintf("%d-%s-%s", t.Year(), str[26:28], str[29:40]))
	cdr.Duration = strings.TrimSpace(str[41:49])
	cdr.Code = strings.TrimSpace(str[50:52])
	cdr.Dialed = strings.TrimSpace(str[53:71])
	cdr.Account = strings.TrimSpace(str[72:89])
	cdr.Cost = strings.TrimSpace(str[90:100])
	cdr.Clid = strings.TrimSpace(str[101:117])
	if n == 154 {
		cdr.Clidname = strings.TrimSpace(str[118:136])
		cdr.Gpno = strings.TrimSpace(str[137:143])
		cdr.Ringtime = strings.TrimSpace(str[143:151])
	} else {
		cdr.Clidname = ""
		cdr.Gpno = ""
		cdr.Ringtime = ""
	}

	return &cdr
}

func skipHeader(line string) string {
	const tag = "===="
	if line[0] == 0x0C {
		n := strings.LastIndex(line, tag)
		if n >= 0 {
			fmt.Print(" HDR ")
			return line[n+len(tag)+2:]
		}
	}
	return line
}

func main() {
	if time.Now().Year() < 2018 {
		panic("System date incorrect")
	}

	loadConfig()

	pabxConn := connectToPABX()
	defer pabxConn.Close()
	pabx := bufio.NewReader(pabxConn)

	ctx := openCallsDatabase()
	defer ctx.Close()

	startWebServer(ctx)

	dumpfile := openDumpFile("dump-file")
	if dumpfile != nil {
		defer dumpfile.Close()
	}

	for {
		// every call record ends with a nul character
		str, err := pabx.ReadString('\000')
		panicErr(err)
		if dumpfile != nil {
			dumpfile.Write([]byte(str))
			dumpfile.Sync()
		}
		fmt.Print(".")

		// every now and then the PABX will preface the call
		// record with a human readable header. Skip it.
		str = skipHeader(str)

		// process this single call record
		cdr := splitData(str)
		if cdr == nil {
			dumpError(str)
		} else {
			insertCDR(ctx, cdr)
		}
	}
}

func dumpError(str string) {
	fmt.Print(" INVALID ")
	errorfile := openDumpFile("error-file")
	if errorfile != nil {
		defer errorfile.Close()
		msg := fmt.Sprintf("\nError: len = %d\n--\n%s\n--\n", len(str), str)
		errorfile.Write([]byte(msg))
		errorfile.Sync()
	}
}

func panicErr(err error, args ...string) {
	if err != nil {
		panic(fmt.Sprintf("Error: %q: %s\n", err, args))
	}
}

/* ----- WEBUI ----- */

func startWebServer(ctx *DbContext) {
	webui := viper.GetString("webui")
	if webui == "" {
		return
	}

	http.HandleFunc("/",
		func(w http.ResponseWriter, r *http.Request) {
			handler(ctx, w, r)
		})

	listener, err := net.Listen("tcp", webui)
	panicErr(err)

	go http.Serve(listener, nil)
}

type Row struct {
	CDR
	Group int
}
type Page struct {
	List []Row
	Prev int
	Next int
}

func handler(ctx *DbContext, w http.ResponseWriter, r *http.Request) {
	folder, err := osext.ExecutableFolder()
	if err != nil {
		fmt.Fprintf(w, "osext = %v", err)
		return
	}
	templateFile := path.Join(folder, "pbxlog.html")

	t := template.New("pbxlog.html")
	t, err = t.ParseFiles(templateFile)
	if err != nil {
		fmt.Fprintf(w, "ParseFiles = %v", err)
		return
	}

	count, err := strconv.Atoi(r.FormValue("count"))
	if err != nil || count < 1 {
		count = 200
	}

	page, err := strconv.Atoi(r.FormValue("page"))
	if err != nil || page < 1 {
		page = 1
	}

	/* not using limit/offset for pagination, see below */
	rows, err := ctx.Db.Query(""+
		"SELECT callid, extension, COALESCE(x.name, '') AS extname, "+
		"auth, COALESCE(a.name, '') AS authname, calltime, duration, "+
		"code, dialed, account, cost, clid, clidname, "+
		"gpno, ringtime "+
		"FROM calls c "+
		"LEFT JOIN extensions x ON c.extension = x.num "+
		"LEFT JOIN extensions a ON c.auth = a.num "+
		"ORDER BY calltime DESC, callid DESC ")
	if err != nil {
		fmt.Fprintf(w, "Query = %v", err)
		return
	}
	defer rows.Close()

	var p Page
	var cdr Row
	var grp map[int]int
	var next int
	var ok bool

	/* handle pagination manually as sqlite3 limit/offset seems to skip
	   records for some reason. Since most people will not be looking too
	   far back in the database manually, although inefficient, this is
	   plenty fast enough */
	skip := (page - 1) * count;
	for i := 0; i < skip; i++ {
		rows.Next()
	}

	grp = make(map[int]int)
	for i := 0; i < count; i++ {
		if !rows.Next() {
			break
		}

		err = rows.Scan(&cdr.Callid, &cdr.Extension, &cdr.ExtName, &cdr.Auth, &cdr.AuthName,
			&cdr.Calltime, &cdr.Duration, &cdr.Code, &cdr.Dialed, &cdr.Account, &cdr.Cost,
			&cdr.Clid, &cdr.Clidname, &cdr.Gpno, &cdr.Ringtime)
		cdr.Group, ok = grp[cdr.Callid]
		if !ok {
			next = (next % 5) + 1
			cdr.Group = next
			grp[cdr.Callid] = cdr.Group
		}
		if err != nil {
			fmt.Fprintf(w, "rows.Scan() = %v", err)
		}
		p.List = append(p.List, cdr)
	}
	err = rows.Err()
	if err != nil {
		fmt.Fprintf(w, "rows.Err() = %v", err)
	}

        p.Prev = page - 1
        if p.Prev < 1 {
 		p.Prev = 1
	}
	p.Next = page + 1

	err = t.Execute(w, p)
	if err != nil {
		fmt.Fprintf(w, "%v", err)
		return
	}
}
<commit_msg>go formatting of the code<commit_after>package main

import (
	"bufio"
	"database/sql"
	"fmt"
	"github.com/kardianos/osext"
	_ "github.com/mattn/go-sqlite3"
	"github.com/spf13/viper"
	"html/template"
	"net"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"time"
)

type DbContext struct {
	Db     *sql.DB
	Insert *sql.Stmt
}

func (c *DbContext) Close() {
	if c.Insert != nil {
		c.Insert.Close()
	}
	if c.Db != nil {
		c.Db.Close()
	}
}

func loadConfig() {
	viper.SetConfigName("pbxlog")
	viper.SetConfigType("yaml")

	viper.AddConfigPath("$HOME")
	viper.AddConfigPath("$HOME/.pbxlog")
	viper.AddConfigPath(".")

	viper.SetDefault("pabx", "")
	viper.SetDefault("webui", "")
	viper.SetDefault("dump-file", "")
	viper.SetDefault("error-file", "")
	viper.SetDefault("calls-db", "")

	err := viper.ReadInConfig()
	if err != nil {
		panic(fmt.Errorf("Fatal error config file: %s \n", err))
	}

	fmt.Printf("Using pabx = %s\n", viper.GetString("pabx"))
	fmt.Printf("Using webui = %s\n", viper.GetString("webui"))
	fmt.Printf("Using dump-file = %s\n", viper.GetString("dump-file"))
	fmt.Printf("Using error-file = %s\n", viper.GetString("error-file"))
	fmt.Printf("Using calls-db = %s\n", viper.GetString("calls-db"))

	if viper.GetString("pabx") == "" || viper.GetString("calls-db") == "" {
		panic("pabx and calls-db configuration values required")
	}
}

func connectToPABX() net.Conn {
	conn, err := net.Dial("tcp", viper.GetString("pabx"))
	panicErr(err)
	return conn
}

func openDumpFile(configItem string) *os.File {
	filename := viper.GetString(configItem)
	if filename == "" {
		return nil
	}

	dumpfile, err := os.OpenFile(filename,
		os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
	panicErr(err)

	return dumpfile
}

type CDR struct {
	Callid    int
	Extension int
	ExtName   string
	Auth      string
	AuthName  string
	Calltime  string
	Duration  string
	Code      string
	Dialed    string
	Account   string
	Cost      string
	Clid      string
	Clidname  string
	Gpno      string
	Ringtime  string
}

func openCallsDatabase() *DbContext {
	ctx := new(DbContext)

	var err error
	ctx.Db, err = sql.Open("sqlite3", viper.GetString("calls-db"))
	panicErr(err)

	_, err = ctx.Db.Exec("CREATE TABLE IF NOT EXISTS calls (" +
		"callid INTEGER, extension INTEGER, auth TEXT, calltime TEXT, duration TEXT, " +
		"code TEXT, dialed TEXT, account TEXT, cost REAL, clid TEXT, " +
		"clidname TEXT, gpno TEXT, ringtime TEXT);")
	panicErr(err)

	_, err = ctx.Db.Exec("CREATE TABLE IF NOT EXISTS extensions (num INTEGER, name TEXT);")
	panicErr(err)

	ctx.Insert, err = ctx.Db.Prepare("INSERT INTO calls(callid, extension, auth, calltime, " +
		"duration, code, dialed, account, cost, clid, clidname, gpno, ringtime) " +
		"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
	panicErr(err)

	return ctx
}

func insertCDR(ctx *DbContext, cdr *CDR) {
	_, err := ctx.Insert.Exec(cdr.Callid, cdr.Extension, cdr.Auth,
		cdr.Calltime, cdr.Duration, cdr.Code, cdr.Dialed, cdr.Account,
		cdr.Cost, cdr.Clid, cdr.Clidname, cdr.Gpno, cdr.Ringtime)
	panicErr(err)
}

func splitData(str string) *CDR {
	n := len(str)
	if n != 154 && n != 122 {
		return nil
	}
	t := time.Now()

	var cdr CDR
	cdr.Callid, _ = strconv.Atoi(strings.TrimSpace(str[2:8]))
	cdr.Extension, _ = strconv.Atoi(strings.TrimSpace(str[9:15]))
	cdr.Auth = strings.TrimSpace(str[16:25])
	cdr.Calltime = strings.TrimSpace(fmt.Sprintf("%d-%s-%s", t.Year(), str[26:28], str[29:40]))
	cdr.Duration = strings.TrimSpace(str[41:49])
	cdr.Code = strings.TrimSpace(str[50:52])
	cdr.Dialed = strings.TrimSpace(str[53:71])
	cdr.Account = strings.TrimSpace(str[72:89])
	cdr.Cost = strings.TrimSpace(str[90:100])
	cdr.Clid = strings.TrimSpace(str[101:117])
	if n == 154 {
		cdr.Clidname = strings.TrimSpace(str[118:136])
		cdr.Gpno = strings.TrimSpace(str[137:143])
		cdr.Ringtime = strings.TrimSpace(str[143:151])
	} else {
		cdr.Clidname = ""
		cdr.Gpno = ""
		cdr.Ringtime = ""
	}

	return &cdr
}

func skipHeader(line string) string {
	const tag = "===="
	if line[0] == 0x0C {
		n := strings.LastIndex(line, tag)
		if n >= 0 {
			fmt.Print(" HDR ")
			return line[n+len(tag)+2:]
		}
	}
	return line
}

func main() {
	if time.Now().Year() < 2018 {
		panic("System date incorrect")
	}

	loadConfig()

	pabxConn := connectToPABX()
	defer pabxConn.Close()
	pabx := bufio.NewReader(pabxConn)

	ctx := openCallsDatabase()
	defer ctx.Close()

	startWebServer(ctx)

	dumpfile := openDumpFile("dump-file")
	if dumpfile != nil {
		defer dumpfile.Close()
	}

	for {
		// every call record ends with a nul character
		str, err := pabx.ReadString('\000')
		panicErr(err)
		if dumpfile != nil {
			dumpfile.Write([]byte(str))
			dumpfile.Sync()
		}
		fmt.Print(".")

		// every now and then the PABX will preface the call
		// record with a human readable header. Skip it.
		str = skipHeader(str)

		// process this single call record
		cdr := splitData(str)
		if cdr == nil {
			dumpError(str)
		} else {
			insertCDR(ctx, cdr)
		}
	}
}

func dumpError(str string) {
	fmt.Print(" INVALID ")
	errorfile := openDumpFile("error-file")
	if errorfile != nil {
		defer errorfile.Close()
		msg := fmt.Sprintf("\nError: len = %d\n--\n%s\n--\n", len(str), str)
		errorfile.Write([]byte(msg))
		errorfile.Sync()
	}
}

func panicErr(err error, args ...string) {
	if err != nil {
		panic(fmt.Sprintf("Error: %q: %s\n", err, args))
	}
}

/* ----- WEBUI ----- */

func startWebServer(ctx *DbContext) {
	webui := viper.GetString("webui")
	if webui == "" {
		return
	}

	http.HandleFunc("/",
		func(w http.ResponseWriter, r *http.Request) {
			handler(ctx, w, r)
		})

	listener, err := net.Listen("tcp", webui)
	panicErr(err)

	go http.Serve(listener, nil)
}

type Row struct {
	CDR
	Group int
}
type Page struct {
	List []Row
	Prev int
	Next int
}

func handler(ctx *DbContext, w http.ResponseWriter, r *http.Request) {
	folder, err := osext.ExecutableFolder()
	if err != nil {
		fmt.Fprintf(w, "osext = %v", err)
		return
	}
	templateFile := path.Join(folder, "pbxlog.html")

	t := template.New("pbxlog.html")
	t, err = t.ParseFiles(templateFile)
	if err != nil {
		fmt.Fprintf(w, "ParseFiles = %v", err)
		return
	}

	count, err := strconv.Atoi(r.FormValue("count"))
	if err != nil || count < 1 {
		count = 200
	}

	page, err := strconv.Atoi(r.FormValue("page"))
	if err != nil || page < 1 {
		page = 1
	}

	/* not using limit/offset for pagination, see below */
	rows, err := ctx.Db.Query("" +
		"SELECT callid, extension, COALESCE(x.name, '') AS extname, " +
		"auth, COALESCE(a.name, '') AS authname, calltime, duration, " +
		"code, dialed, account, cost, clid, clidname, " +
		"gpno, ringtime " +
		"FROM calls c " +
		"LEFT JOIN extensions x ON c.extension = x.num " +
		"LEFT JOIN extensions a ON c.auth = a.num " +
		"ORDER BY calltime DESC, callid DESC ")
	if err != nil {
		fmt.Fprintf(w, "Query = %v", err)
		return
	}
	defer rows.Close()

	var p Page
	var cdr Row
	var grp map[int]int
	var next int
	var ok bool

	/* handle pagination manually as sqlite3 limit/offset seems to skip
	   records for some reason. Since most people will not be looking too
	   far back in the database manually, although inefficient, this is
	   plenty fast enough */
	skip := (page - 1) * count
	for i := 0; i < skip; i++ {
		rows.Next()
	}

	grp = make(map[int]int)
	for i := 0; i < count; i++ {
		if !rows.Next() {
			break
		}

		err = rows.Scan(&cdr.Callid, &cdr.Extension, &cdr.ExtName, &cdr.Auth, &cdr.AuthName,
			&cdr.Calltime, &cdr.Duration, &cdr.Code, &cdr.Dialed, &cdr.Account, &cdr.Cost,
			&cdr.Clid, &cdr.Clidname, &cdr.Gpno, &cdr.Ringtime)
		cdr.Group, ok = grp[cdr.Callid]
		if !ok {
			next = (next % 5) + 1
			cdr.Group = next
			grp[cdr.Callid] = cdr.Group
		}
		if err != nil {
			fmt.Fprintf(w, "rows.Scan() = %v", err)
		}
		p.List = append(p.List, cdr)
	}
	err = rows.Err()
	if err != nil {
		fmt.Fprintf(w, "rows.Err() = %v", err)
	}

	p.Prev = page - 1
	if p.Prev < 1 {
		p.Prev = 1
	}
	p.Next = page + 1

	err = t.Execute(w, p)
	if err != nil {
		fmt.Fprintf(w, "%v", err)
		return
	}
}
<|endoftext|>


<commit_before>package cloudtest

import (
	"github.com/aws/aws-sdk-go/service/lambda"
	"github.com/pkg/errors"
)

// stackLambdaLiteralSelector returns a lambda function
type stackLambdaLiteralSelector struct {
	literalName string
}

func (slls *stackLambdaLiteralSelector) Select(t CloudTest) (*lambda.GetFunctionOutput, error) {
	output := cache.getFunction(t, slls.literalName)
	if output == nil {
		return nil, errors.Errorf("Failed to find named function: %s", slls.literalName)
	}
	return output, nil
}

// NewLambdaLiteralSelector returns a new LambdaSelector that just uses
// the hardcoded name
func NewLambdaLiteralSelector(functionName string) LambdaSelector {
	return &stackLambdaLiteralSelector{
		literalName: functionName,
	}
}

// stackLambdasSelector returns a lambda function
type stackLambdasSelector struct {
	stackName    string
	jmesSelector string
}

func (sls *stackLambdasSelector) Select(t CloudTest) (*lambda.GetFunctionOutput, error) {

	functionOutput := cache.getStackFunction(t, sls.stackName, sls.jmesSelector)
	if functionOutput == nil {
		return nil, errors.Errorf("Failed to find AWS Laambda in stack %s for selector: %s",
			sls.stackName,
			sls.jmesSelector)
	}
	return functionOutput, nil
}

// NewStackLambdaSelector returns a new LambdaSelector using the jmesSelector
// expression against all GetFunctionOutputs for that lambda function
func NewStackLambdaSelector(stackName string, jmesSelector string) LambdaSelector {
	return &stackLambdasSelector{
		stackName:    stackName,
		jmesSelector: jmesSelector,
	}
}
<commit_msg>Correct typo<commit_after>package cloudtest

import (
	"github.com/aws/aws-sdk-go/service/lambda"
	"github.com/pkg/errors"
)

// stackLambdaLiteralSelector returns a lambda function
type stackLambdaLiteralSelector struct {
	literalName string
}

func (slls *stackLambdaLiteralSelector) Select(t CloudTest) (*lambda.GetFunctionOutput, error) {
	output := cache.getFunction(t, slls.literalName)
	if output == nil {
		return nil, errors.Errorf("Failed to find named function: %s", slls.literalName)
	}
	return output, nil
}

// NewLambdaLiteralSelector returns a new LambdaSelector that just uses
// the hardcoded name
func NewLambdaLiteralSelector(functionName string) LambdaSelector {
	return &stackLambdaLiteralSelector{
		literalName: functionName,
	}
}

// stackLambdasSelector returns a lambda function
type stackLambdasSelector struct {
	stackName    string
	jmesSelector string
}

func (sls *stackLambdasSelector) Select(t CloudTest) (*lambda.GetFunctionOutput, error) {

	functionOutput := cache.getStackFunction(t, sls.stackName, sls.jmesSelector)
	if functionOutput == nil {
		return nil, errors.Errorf("Failed to find AWS Lambda in stack %s for selector: %s",
			sls.stackName,
			sls.jmesSelector)
	}
	return functionOutput, nil
}

// NewStackLambdaSelector returns a new LambdaSelector using the jmesSelector
// expression against all GetFunctionOutputs for that lambda function
func NewStackLambdaSelector(stackName string, jmesSelector string) LambdaSelector {
	return &stackLambdasSelector{
		stackName:    stackName,
		jmesSelector: jmesSelector,
	}
}
<|endoftext|>


<commit_before>package gstruct_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gstruct"
)

var _ = Describe("Struct", func() {
	allFields := struct{ A, B string }{"a", "b"}
	missingFields := struct{ A string }{"a"}
	extraFields := struct{ A, B, C string }{"a", "b", "c"}
	emptyFields := struct{ A, B string }{}

	It("should strictly match all fields", func() {
		m := MatchAllFields(Fields{
			"B": Equal("b"),
			"A": Equal("a"),
		})
		Expect(allFields).Should(m, "should match all fields")
		Expect(missingFields).ShouldNot(m, "should fail with missing fields")
		Expect(extraFields).ShouldNot(m, "should fail with extra fields")
		Expect(emptyFields).ShouldNot(m, "should fail with empty fields")

		m = MatchAllFields(Fields{
			"A": Equal("a"),
			"B": Equal("fail"),
		})
		Expect(allFields).ShouldNot(m, "should run nested matchers")
	})

	It("should handle empty structs", func() {
		m := MatchAllFields(Fields{})
		Expect(struct{}{}).Should(m, "should handle empty structs")
		Expect(allFields).ShouldNot(m, "should fail with extra fields")
	})

	It("should ignore missing fields", func() {
		m := MatchFields(IgnoreMissing, Fields{
			"B": Equal("b"),
			"A": Equal("a"),
		})
		Expect(allFields).Should(m, "should match all fields")
		Expect(missingFields).Should(m, "should ignore missing fields")
		Expect(extraFields).ShouldNot(m, "should fail with extra fields")
		Expect(emptyFields).ShouldNot(m, "should fail with empty fields")
	})

	It("should ignore extra fields", func() {
		m := MatchFields(IgnoreExtras, Fields{
			"B": Equal("b"),
			"A": Equal("a"),
		})
		Expect(allFields).Should(m, "should match all fields")
		Expect(missingFields).ShouldNot(m, "should fail with missing fields")
		Expect(extraFields).Should(m, "should ignore extra fields")
		Expect(emptyFields).ShouldNot(m, "should fail with empty fields")
	})

	It("should ignore missing and extra fields", func() {
		m := MatchFields(IgnoreMissing|IgnoreExtras, Fields{
			"B": Equal("b"),
			"A": Equal("a"),
		})
		Expect(allFields).Should(m, "should match all fields")
		Expect(missingFields).Should(m, "should ignore missing fields")
		Expect(extraFields).Should(m, "should ignore extra fields")
		Expect(emptyFields).ShouldNot(m, "should fail with empty fields")

		m = MatchFields(IgnoreMissing|IgnoreExtras, Fields{
			"A": Equal("a"),
			"B": Equal("fail"),
		})
		Expect(allFields).ShouldNot(m, "should run nested matchers")
	})
})
<commit_msg>add analogous tests to fields_test.go<commit_after>package gstruct_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gstruct"
)

var _ = Describe("Struct", func() {
	allFields := struct{ A, B string }{"a", "b"}
	missingFields := struct{ A string }{"a"}
	extraFields := struct{ A, B, C string }{"a", "b", "c"}
	emptyFields := struct{ A, B string }{}

	It("should strictly match all fields", func() {
		m := MatchAllFields(Fields{
			"B": Equal("b"),
			"A": Equal("a"),
		})
		Expect(allFields).Should(m, "should match all fields")
		Expect(missingFields).ShouldNot(m, "should fail with missing fields")
		Expect(extraFields).ShouldNot(m, "should fail with extra fields")
		Expect(emptyFields).ShouldNot(m, "should fail with empty fields")

		m = MatchAllFields(Fields{
			"A": Equal("a"),
			"B": Equal("fail"),
		})
		Expect(allFields).ShouldNot(m, "should run nested matchers")
	})

	It("should handle empty structs", func() {
		m := MatchAllFields(Fields{})
		Expect(struct{}{}).Should(m, "should handle empty structs")
		Expect(allFields).ShouldNot(m, "should fail with extra fields")
	})

	It("should ignore missing fields", func() {
		m := MatchFields(IgnoreMissing, Fields{
			"B": Equal("b"),
			"A": Equal("a"),
		})
		Expect(allFields).Should(m, "should match all fields")
		Expect(missingFields).Should(m, "should ignore missing fields")
		Expect(extraFields).ShouldNot(m, "should fail with extra fields")
		Expect(emptyFields).ShouldNot(m, "should fail with empty fields")
	})

	It("should ignore extra fields", func() {
		m := MatchFields(IgnoreExtras, Fields{
			"B": Equal("b"),
			"A": Equal("a"),
		})
		Expect(allFields).Should(m, "should match all fields")
		Expect(missingFields).ShouldNot(m, "should fail with missing fields")
		Expect(extraFields).Should(m, "should ignore extra fields")
		Expect(emptyFields).ShouldNot(m, "should fail with empty fields")
	})

	It("should ignore missing and extra fields", func() {
		m := MatchFields(IgnoreMissing|IgnoreExtras, Fields{
			"B": Equal("b"),
			"A": Equal("a"),
		})
		Expect(allFields).Should(m, "should match all fields")
		Expect(missingFields).Should(m, "should ignore missing fields")
		Expect(extraFields).Should(m, "should ignore extra fields")
		Expect(emptyFields).ShouldNot(m, "should fail with empty fields")

		m = MatchFields(IgnoreMissing|IgnoreExtras, Fields{
			"A": Equal("a"),
			"B": Equal("fail"),
		})
		Expect(allFields).ShouldNot(m, "should run nested matchers")
	})

	It("should produce sensible error messages", func() {
		m := MatchAllFields(Fields{
			"B": Equal("b"),
			"A": Equal("a"),
		})

		actual := struct{ A, C string }{A: "b", C: "c"}

		//Because the order of the constituent errors can't be guaranteed,
		//we do a number of checks to make sure everything's included
		m.Match(actual)
		Expect(m.FailureMessage(actual)).Should(HavePrefix(
			"Expected\n    <string>: \nto match fields: {\n",
		))
		Expect(m.FailureMessage(actual)).Should(ContainSubstring(
			".A:\n	Expected\n	    <string>: b\n	to equal\n	    <string>: a\n",
		))
		Expect(m.FailureMessage(actual)).Should(ContainSubstring(
			"missing expected field B\n",
		))
		Expect(m.FailureMessage(actual)).Should(ContainSubstring(
			".C:\n	unexpected field C: {A:b C:c}",
		))
	})
})
<|endoftext|>


<commit_before>package main

import (
	"github.com/jkomoros/sudoku"
)

type model struct {
	grid *sudoku.Grid
}

//TODO: rename Mutator to command
type command interface {
	Apply(m *model)
	Undo(m *model)
}

type markCommand struct {
	row, col    int
	marksToggle map[int]bool
}

type numberCommand struct {
	row, col int
	number   int
	//Necessary so we can undo.
	oldNumber int
}

func (m *model) SetGrid(grid *sudoku.Grid) {
	m.grid = grid
}

func (m *model) SetMarks(row, col int, marksToggle map[int]bool) {
	command := m.newMarkCommand(row, col, marksToggle)
	command.Apply(m)
}

func (m *model) newMarkCommand(row, col int, marksToggle map[int]bool) *markCommand {
	//Only keep marks in the toggle that won't be a no-op
	newMarksToggle := make(map[int]bool)

	cell := m.grid.Cell(row, col)

	if cell == nil {
		return nil
	}
	for key, value := range marksToggle {
		if cell.Mark(key) != value {
			//Good, keep it
			newMarksToggle[key] = value
		}
	}

	if len(newMarksToggle) == 0 {
		//The command would be a no op!
		return nil
	}

	return &markCommand{row, col, newMarksToggle}
}

func (m *model) SetNumber(row, col int, num int) {
	//TODO: this should also only add an action if the specified cell is not already num.
	cell := m.grid.Cell(row, col)
	if cell == nil {
		return
	}
	mutator := &numberCommand{row, col, num, cell.Number()}
	mutator.Apply(m)
}

//TODO: implement model.new{Mark|Number}Mutator, which only return a
//modelMutator if it wouldn't be a no-op. Then, test that they return nil if
//it would be a no op, including omitting marks that would be a no op.

func (m *markCommand) Apply(model *model) {
	cell := model.grid.Cell(m.row, m.col)
	if cell == nil {
		return
	}
	for key, value := range m.marksToggle {
		cell.SetMark(key, value)
	}
}

func (m *markCommand) Undo(model *model) {
	cell := model.grid.Cell(m.row, m.col)
	if cell == nil {
		return
	}
	for key, value := range m.marksToggle {
		//Set the opposite since we're undoing.
		cell.SetMark(key, !value)
	}
}

func (n *numberCommand) Apply(model *model) {
	cell := model.grid.Cell(n.row, n.col)
	if cell == nil {
		return
	}
	cell.SetNumber(n.number)
}

func (n *numberCommand) Undo(model *model) {
	cell := model.grid.Cell(n.row, n.col)
	if cell == nil {
		return
	}
	cell.SetNumber(n.oldNumber)
}
<commit_msg>Fixed potentil nil method error<commit_after>package main

import (
	"github.com/jkomoros/sudoku"
)

type model struct {
	grid *sudoku.Grid
}

//TODO: rename Mutator to command
type command interface {
	Apply(m *model)
	Undo(m *model)
}

type markCommand struct {
	row, col    int
	marksToggle map[int]bool
}

type numberCommand struct {
	row, col int
	number   int
	//Necessary so we can undo.
	oldNumber int
}

func (m *model) SetGrid(grid *sudoku.Grid) {
	m.grid = grid
}

func (m *model) SetMarks(row, col int, marksToggle map[int]bool) {
	command := m.newMarkCommand(row, col, marksToggle)
	if command == nil {
		return
	}
	command.Apply(m)
}

func (m *model) newMarkCommand(row, col int, marksToggle map[int]bool) *markCommand {
	//Only keep marks in the toggle that won't be a no-op
	newMarksToggle := make(map[int]bool)

	cell := m.grid.Cell(row, col)

	if cell == nil {
		return nil
	}
	for key, value := range marksToggle {
		if cell.Mark(key) != value {
			//Good, keep it
			newMarksToggle[key] = value
		}
	}

	if len(newMarksToggle) == 0 {
		//The command would be a no op!
		return nil
	}

	return &markCommand{row, col, newMarksToggle}
}

func (m *model) SetNumber(row, col int, num int) {
	//TODO: this should also only add an action if the specified cell is not already num.
	cell := m.grid.Cell(row, col)
	if cell == nil {
		return
	}
	mutator := &numberCommand{row, col, num, cell.Number()}
	mutator.Apply(m)
}

//TODO: implement model.new{Mark|Number}Mutator, which only return a
//modelMutator if it wouldn't be a no-op. Then, test that they return nil if
//it would be a no op, including omitting marks that would be a no op.

func (m *markCommand) Apply(model *model) {
	cell := model.grid.Cell(m.row, m.col)
	if cell == nil {
		return
	}
	for key, value := range m.marksToggle {
		cell.SetMark(key, value)
	}
}

func (m *markCommand) Undo(model *model) {
	cell := model.grid.Cell(m.row, m.col)
	if cell == nil {
		return
	}
	for key, value := range m.marksToggle {
		//Set the opposite since we're undoing.
		cell.SetMark(key, !value)
	}
}

func (n *numberCommand) Apply(model *model) {
	cell := model.grid.Cell(n.row, n.col)
	if cell == nil {
		return
	}
	cell.SetNumber(n.number)
}

func (n *numberCommand) Undo(model *model) {
	cell := model.grid.Cell(n.row, n.col)
	if cell == nil {
		return
	}
	cell.SetNumber(n.oldNumber)
}
<|endoftext|>


<commit_before>package photon

import (
	"net/http"

	"github.com/as27/govuegui"
	"github.com/as27/govuegui/gui/photon/src/pkg/photoncss"
	"github.com/as27/govuegui/gui/photon/src/pkg/photonwoff"
)

var Template = govuegui.GuiTemplate{
	CSSHandler: photoncss.Handler,
	CustomCSS: `#govuegui{
        max-width:1200px;
        margin: auto;
    }`,
	Files: map[string]func(w http.ResponseWriter, r *http.Request){
		"photon-entypo.woff": photonwoff.Handler,
	},
	Body: `

{{appTitle}}

                <router-link
                    v-for="form in data.Forms"
                    active-class="active"
                    class="btn btn-default"
                    tag="button"
                    :to="{name: 'gvgform', params: { formid: form.id}}">

                    {{form.id}}</router-link>

            <router-view :data=data :forms=forms ></router-view>

        govuigui
        by Andreas Schröpfer
      | govuigui github page

`,
	GvgForms: `

Forms

            <router-link
                v-for="form in data.Forms"
                active-class="active"
                class="nav-group-item"
                :to="{name: 'gvgform', params: { formid: form.id}}">

                {{form.id}}</router-link>

            <router-view :data=data :form=forms[formid] :formid=formid></router-view>

  `,

	GvgForm: `

    <router-link v-for="box in form.Boxes"
        active-class="active"
        class="tab-item"
        tag="div"
        :to="{ name: 'gvgbox', params: { boxid: box.id}}">
        {{box.id}}
    </router-link>

    <gvgbox :box=myBox :data=data></gvgbox>
    <button class="btn btn-large btn-primary" @click="saveData">Submit</button>

`,

	GvgBox: `<form class="padded-more">

{{box.id}}

            <gvgelement
                v-for="element in box.elements"
                :element=element
                :data=data></gvgelement>
    </form>
   `,

	GvgElement: `

        {{element.label}}
        <component :is=element.type :element=element :data=data v-model="data.Data.data[element.id]"></component>

`,
	GvgButton: `<div><br><button class="btn btn-large btn-primary" @click="callAction">{{element.label}}</button><br></div>`,

	GvgList: `

{{litem}}

`,
	GvgDropdown: `

    <select v-model="data.Data.data[element.id]" class="form-control">
    <option v-for="oitem in element.options" v-bind:value="oitem.Option">{{oitem.Values[0]}}</option>
    </select>

`,

	GvgTable: `

{{cell}}

 0">
    {{cell}}

`,
	GvgText: `<div class="text" v-html="data.Data.data[element.id]"></div>`,

	GvgTextarea: `<textarea class="form-control" v-model="data.Data.data[element.id]"></textarea>`,

	GvgInput: `<input class="form-control" type="text" v-model="data.Data.data[element.id]">`,
}
<commit_msg>photon template changed<commit_after>package photon

import (
	"net/http"

	"github.com/as27/govuegui"
	"github.com/as27/govuegui/gui/photon/src/pkg/photoncss"
	"github.com/as27/govuegui/gui/photon/src/pkg/photonwoff"
)

var Template = govuegui.GuiTemplate{
	CSSHandler: photoncss.Handler,
	CustomCSS: `#govuegui{
        max-width:1200px;
        margin: auto;
    }`,
	Files: map[string]func(w http.ResponseWriter, r *http.Request){
		"photon-entypo.woff": photonwoff.Handler,
	},
	Body: `

{{appTitle}}

                <router-link
                    v-for="form in data.Forms"
                    active-class="active"
                    class="btn btn-default"
                    tag="button"
                    :to="{name: 'gvgform', params: { formid: form.id}}">

                    {{form.id}}</router-link>

            <router-view :data=data :forms=forms ></router-view>

        govuigui
        by Andreas Schröpfer
      | govuigui github page

`,
	GvgForms: `

            <router-view :data=data :form=forms[formid] :formid=formid></router-view>

  `,

	GvgForm: `

Boxes

            <router-link
                v-for="box in form.Boxes"
                active-class="active"
                class="nav-group-item"
                :to="{ name: 'gvgbox', params: { boxid: box.id}}">

                {{box.id}}</router-link>

        <gvgbox :box=myBox :data=data></gvgbox>
        <button class="btn btn-large btn-primary" @click="saveData">Submit</button>

    `,

	GvgBox: `<form class="padded-more">

{{box.id}}

            <gvgelement
                v-for="element in box.elements"
                :element=element
                :data=data></gvgelement>
    </form>
   `,

	GvgElement: `

        {{element.label}}
        <component :is=element.type :element=element :data=data v-model="data.Data.data[element.id]"></component>

`,
	GvgButton: `<div><br><button class="btn btn-large btn-primary" @click="callAction">{{element.label}}</button><br></div>`,

	GvgList: `

{{litem}}

`,
	GvgDropdown: `

    <select v-model="data.Data.data[element.id]" class="form-control">
    <option v-for="oitem in element.options" v-bind:value="oitem.Option">{{oitem.Values[0]}}</option>
    </select>

`,

	GvgTable: `

{{cell}}

 0">
    {{cell}}

`,
	GvgText: `<div class="text" v-html="data.Data.data[element.id]"></div>`,

	GvgTextarea: `<textarea class="form-control" v-model="data.Data.data[element.id]"></textarea>`,

	GvgInput: `<input class="form-control" type="text" v-model="data.Data.data[element.id]">`,
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"

	"github.com/raintreeinc/knowledgebase/kbserver/pgdb"

	_ "github.com/lib/pq"
)

var (
	database = flag.String("database", "user=root dbname=knowledgebase sslmode=disable", "database `params`")
)

func main() {
	flag.Parse()
	if os.Getenv("DATABASE") != "" {
		*database = os.Getenv("DATABASE")
	}

	ditamap := os.Getenv("DITAMAP")

	// Load database
	db, err := pgdb.New(*database)
	if err != nil {
		log.Fatal(err)
	}

	users, err := db.Users().List()
	log.Println(err)
	for _, user := range users {
		log.Printf("%+v\n", user)
	}

	//user, err := db.Users().ByID("usernaem")
}
<commit_msg>Remove unneeded var.<commit_after>package main

import (
	"flag"
	"log"
	"os"

	"github.com/raintreeinc/knowledgebase/kbserver/pgdb"

	_ "github.com/lib/pq"
)

var (
	database = flag.String("database", "user=root dbname=knowledgebase sslmode=disable", "database `params`")
)

func main() {
	flag.Parse()
	if os.Getenv("DATABASE") != "" {
		*database = os.Getenv("DATABASE")
	}

	// Load database
	db, err := pgdb.New(*database)
	if err != nil {
		log.Fatal(err)
	}

	users, err := db.Users().List()
	log.Println(err)
	for _, user := range users {
		log.Printf("%+v\n", user)
	}
}
<|endoftext|>


<commit_before>/*
Command lmdb_stat is a clone of mdb_stat that displays the status an LMDB
environment.

Command line flags mirror the flags for the original program.  For information
about, run lmdb_stat with the -h flag.

	lmdb_stat -h
*/
package main

import "C"

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"reflect"
	"strings"
	"unsafe"

	"github.com/bmatsuo/lmdb-go/exp/lmdbscan"
	"github.com/bmatsuo/lmdb-go/internal/lmdbcmd"
	"github.com/bmatsuo/lmdb-go/lmdb"
)

func main() {
	opt := &Options{}
	flag.BoolVar(&opt.PrintInfo, "e", false, "Display information about the database environment")
	flag.BoolVar(&opt.PrintFree, "f", false, "Display freelist information")
	flag.BoolVar(&opt.PrintFreeSummary, "ff", false, "Display freelist information")
	flag.BoolVar(&opt.PrintFreeFull, "fff", false, "Display freelist information")
	flag.BoolVar(&opt.PrintReaders, "r", false, strings.Join([]string{
		"Display information about the environment reader table.",
		"Shows the process ID, thread ID, and transaction ID for each active reader slot.",
	}, "  "))
	flag.BoolVar(&opt.PrintReadersCheck, "rr", false, strings.Join([]string{
		"Implies -r.",
		"Check for stale entries in the reader table and clear them.",
		"The reader table is printed again after the check is performed.",
	}, "  "))
	flag.BoolVar(&opt.PrintStatAll, "a", false, "Display the status of all databases in the environment")
	flag.StringVar(&opt.PrintStatSub, "s", "", "Display the status of a specific subdatabase.")
	flag.BoolVar(&opt.Debug, "D", false, "print debug information")
	flag.Parse()

	lmdbcmd.PrintVersion()

	if opt.PrintStatAll && opt.PrintStatSub != "" {
		log.Fatal("only one of -a and -s may be provided")
	}

	if flag.NArg() > 1 {
		log.Fatalf("too many argument provided")
	}
	if flag.NArg() == 0 {
		log.Fatalf("missing argument")
	}
	opt.Path = flag.Arg(0)

	var failed bool
	defer func() {
		if e := recover(); e != nil {
			if opt.Debug {
				panic(e)
			}
			log.Print(e)
			failed = true
		}
		if failed {
			os.Exit(1)
		}
	}()

	err := doMain(opt)
	if err != nil {
		log.Print(err)
		failed = true
	}
}

// Options contains all the configuration for an lmdb_stat command including
// command line arguments.
type Options struct {
	Path string

	PrintInfo         bool
	PrintReaders      bool
	PrintReadersCheck bool
	PrintFree         bool
	PrintFreeSummary  bool
	PrintFreeFull     bool
	PrintStatAll      bool
	PrintStatSub      string

	Debug bool
}

func doMain(opt *Options) error {
	env, err := lmdb.NewEnv()
	if err != nil {
		return err
	}
	if opt.PrintStatAll || opt.PrintStatSub != "" {
		err = env.SetMaxDBs(1)
		if err != nil {
			return err
		}
	}
	err = env.Open(opt.Path, lmdbcmd.OpenFlag(), 0644)
	defer env.Close()
	if err != nil {
		return err
	}

	if opt.PrintInfo {
		err = doPrintInfo(env, opt)
		if err != nil {
			return err
		}
	}

	if opt.PrintReaders || opt.PrintReadersCheck {
		err = doPrintReaders(env, opt)
		if err != nil {
			return err
		}
	}

	if opt.PrintFree || opt.PrintFreeSummary || opt.PrintFreeFull {
		err = doPrintFree(env, opt)
		if err != nil {
			return err
		}
	}

	err = doPrintStatRoot(env, opt)
	if err != nil {
		return err
	}

	if opt.PrintStatAll {
		err = doPrintStatAll(env, opt)
		if err != nil {
			return err
		}
	} else if opt.PrintStatSub != "" {
		err = doPrintStatDB(env, opt.PrintStatSub, opt)
		if err != nil {
			return err
		}
	}

	return nil
}

func doPrintInfo(env *lmdb.Env, opt *Options) error {
	info, err := env.Info()
	if err != nil {
		return err
	}

	pagesize := os.Getpagesize()

	fmt.Println("Environment Info")
	fmt.Println("  Map address:", nil)
	fmt.Println("  Map size:", info.MapSize)
	fmt.Println("  Page size:", pagesize)
	fmt.Println("  Max pages:", info.MapSize/int64(pagesize))
	fmt.Println("  Number of pages used:", info.LastPNO+1)
	fmt.Println("  Last transaction ID:", info.LastTxnID)
	fmt.Println("  Max readers:", info.MaxReaders)
	fmt.Println("  Number of readers used:", info.NumReaders)

	return nil
}

func doPrintReaders(env *lmdb.Env, opt *Options) error {
	fmt.Println("Reader Table Status")
	w := bufio.NewWriter(os.Stdout)
	err := printReaders(env, w, opt)
	if err == nil {
		err = w.Flush()
	}
	if err != nil {
		return err
	}

	if opt.PrintReadersCheck {
		numstale, err := env.ReaderCheck()
		if err != nil {
			return err
		}
		fmt.Printf("  %d stale readers cleared.\n", numstale)
		err = printReaders(env, w, opt)
		if err == nil {
			err = w.Flush()
		}
		if err != nil {
			return err
		}
	}

	return nil
}

func printReaders(env *lmdb.Env, w io.Writer, opt *Options) error {
	return env.ReaderList(func(msg string) error {
		_, err := fmt.Fprint(w, msg)
		return err
	})
}

func doPrintFree(env *lmdb.Env, opt *Options) error {
	return env.View(func(txn *lmdb.Txn) (err error) {
		txn.RawRead = true

		fmt.Println("Freelist status")

		stat, err := txn.Stat(0)
		if err != nil {
			return err
		}
		printStat(stat, opt)

		var numpages int64
		s := lmdbscan.New(txn, 0)
		defer s.Close()
		for s.Scan() {
			key := s.Key()
			data := s.Val()
			txid := *(*C.size_t)(unsafe.Pointer(&key[0]))
			ipages := int64(*(*C.size_t)(unsafe.Pointer(&data[0])))
			numpages += ipages
			if opt.PrintFreeSummary || opt.PrintFreeFull {
				bad := ""
				hdr := reflect.SliceHeader{
					Data: uintptr(unsafe.Pointer(&data[0])),
					Len:  int(ipages) + 1,
					Cap:  int(ipages) + 1,
				}
				pages := *(*[]C.size_t)(unsafe.Pointer(&hdr))
				pages = pages[1:]
				var span C.size_t
				prev := C.size_t(1)
				for i := ipages - 1; i >= 0; i-- {
					pg := pages[i]
					if pg < prev {
						bad = " [bad sequence]"
					}
					prev = pg
					pg += span
					for i >= int64(span) && pages[i-int64(span)] == pg {
						span++
						pg++
					}
				}
				fmt.Printf("    Transaction %x, %d pages, maxspan %d%s\n", txid, ipages, span, bad)

				if opt.PrintFreeFull {
					for j := ipages - 1; j >= 0; {
						pg := pages[j]
						j--
						span := C.size_t(1)
						for j >= 0 && pages[j] == pg+span {
							j--
							span++
						}
						if span > 1 {
							fmt.Printf("     %9x[%d]\n", pg, span)
						} else {
							fmt.Printf("     %9x\n", pg)
						}
					}
				}
			}
		}
		err = s.Err()
		if err != nil {
			return err
		}

		fmt.Println("  Free pages:", numpages)

		return nil
	})
}

func doPrintStatRoot(env *lmdb.Env, opt *Options) error {
	stat, err := env.Stat()
	if err != nil {
		return err
	}

	fmt.Println("Status of Main DB")
	fmt.Println("  Tree depth:", stat.Depth)
	fmt.Println("  Branch pages:", stat.BranchPages)
	fmt.Println("  Lead pages:", stat.LeafPages)
	fmt.Println("  Overflow pages:", stat.OverflowPages)
	fmt.Println("  Entries:", stat.Entries)

	return nil
}

func doPrintStatDB(env *lmdb.Env, db string, opt *Options) error {
	err := env.View(func(txn *lmdb.Txn) (err error) {
		return printStatDB(env, txn, db, opt)
	})
	if err != nil {
		return fmt.Errorf("%v (%s)", err, db)
	}
	return nil
}

func printStatDB(env *lmdb.Env, txn *lmdb.Txn, db string, opt *Options) error {
	dbi, err := txn.OpenDBI(db, 0)
	if err != nil {
		return err
	}
	defer env.CloseDBI(dbi)

	stat, err := txn.Stat(dbi)
	if err != nil {
		return err
	}

	fmt.Println("Status of", db)
	printStat(stat, opt)

	return err
}

func printStat(stat *lmdb.Stat, opt *Options) error {
	fmt.Println("  Tree depth:", stat.Depth)
	fmt.Println("  Branch pages:", stat.BranchPages)
	fmt.Println("  Lead pages:", stat.LeafPages)
	fmt.Println("  Overflow pages:", stat.OverflowPages)
	fmt.Println("  Entries:", stat.Entries)

	return nil
}

func doPrintStatAll(env *lmdb.Env, opt *Options) error {
	return env.View(func(txn *lmdb.Txn) (err error) {
		dbi, err := txn.OpenRoot(0)
		if err != nil {
			return err
		}
		defer env.CloseDBI(dbi)

		s := lmdbscan.New(txn, dbi)
		defer s.Close()
		for s.Scan() {
			err = printStatDB(env, txn, string(s.Key()), opt)
			if e, ok := err.(*lmdb.OpError); ok {
				if e.Op == "mdb_dbi_open" {
					continue
				}
			}
			if err != nil {
				return fmt.Errorf("%v (%s)", err, s.Key())
			}
		}
		return s.Err()
	})
}
<commit_msg>cmd/lmdb_stat: Fix typos and format flags to be 1:1 with mdb_stat<commit_after>/*
Command lmdb_stat is a clone of mdb_stat that displays the status an LMDB
environment.

Command line flags mirror the flags for the original program.  For information
about, run lmdb_stat with the -h flag.

	lmdb_stat -h
*/
package main

import "C"

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"reflect"
	"strings"
	"unsafe"

	"github.com/bmatsuo/lmdb-go/exp/lmdbscan"
	"github.com/bmatsuo/lmdb-go/internal/lmdbcmd"
	"github.com/bmatsuo/lmdb-go/lmdb"
)

func main() {
	opt := &Options{}
	flag.BoolVar(&opt.PrintInfo, "e", false, "Display information about the database environment")
	flag.BoolVar(&opt.PrintFree, "f", false, "Display freelist information")
	flag.BoolVar(&opt.PrintFreeSummary, "ff", false, "Display freelist information")
	flag.BoolVar(&opt.PrintFreeFull, "fff", false, "Display freelist information")
	flag.BoolVar(&opt.PrintReaders, "r", false, strings.Join([]string{
		"Display information about the environment reader table.",
		"Shows the process ID, thread ID, and transaction ID for each active reader slot.",
	}, "  "))
	flag.BoolVar(&opt.PrintReadersCheck, "rr", false, strings.Join([]string{
		"Implies -r.",
		"Check for stale entries in the reader table and clear them.",
		"The reader table is printed again after the check is performed.",
	}, "  "))
	flag.BoolVar(&opt.PrintStatAll, "a", false, "Display the status of all databases in the environment")
	flag.StringVar(&opt.PrintStatSub, "s", "", "Display the status of a specific subdatabase.")
	flag.BoolVar(&opt.Debug, "D", false, "print debug information")
	flag.Parse()

	lmdbcmd.PrintVersion()

	if opt.PrintStatAll && opt.PrintStatSub != "" {
		log.Fatal("only one of -a and -s may be provided")
	}

	if flag.NArg() > 1 {
		log.Fatalf("too many argument provided")
	}
	if flag.NArg() == 0 {
		log.Fatalf("missing argument")
	}
	opt.Path = flag.Arg(0)

	var failed bool
	defer func() {
		if e := recover(); e != nil {
			if opt.Debug {
				panic(e)
			}
			log.Print(e)
			failed = true
		}
		if failed {
			os.Exit(1)
		}
	}()

	err := doMain(opt)
	if err != nil {
		log.Print(err)
		failed = true
	}
}

// Options contains all the configuration for an lmdb_stat command including
// command line arguments.
type Options struct {
	Path string

	PrintInfo         bool
	PrintReaders      bool
	PrintReadersCheck bool
	PrintFree         bool
	PrintFreeSummary  bool
	PrintFreeFull     bool
	PrintStatAll      bool
	PrintStatSub      string

	Debug bool
}

func doMain(opt *Options) error {
	env, err := lmdb.NewEnv()
	if err != nil {
		return err
	}
	if opt.PrintStatAll || opt.PrintStatSub != "" {
		err = env.SetMaxDBs(1)
		if err != nil {
			return err
		}
	}
	err = env.Open(opt.Path, lmdbcmd.OpenFlag(), 0644)
	defer env.Close()
	if err != nil {
		return err
	}

	if opt.PrintInfo {
		err = doPrintInfo(env, opt)
		if err != nil {
			return err
		}
	}

	if opt.PrintReaders || opt.PrintReadersCheck {
		err = doPrintReaders(env, opt)
		if err != nil {
			return err
		}
	}

	if opt.PrintFree || opt.PrintFreeSummary || opt.PrintFreeFull {
		err = doPrintFree(env, opt)
		if err != nil {
			return err
		}
	}

	err = doPrintStatRoot(env, opt)
	if err != nil {
		return err
	}

	if opt.PrintStatAll {
		err = doPrintStatAll(env, opt)
		if err != nil {
			return err
		}
	} else if opt.PrintStatSub != "" {
		err = doPrintStatDB(env, opt.PrintStatSub, opt)
		if err != nil {
			return err
		}
	}

	return nil
}

func doPrintInfo(env *lmdb.Env, opt *Options) error {
	info, err := env.Info()
	if err != nil {
		return err
	}

	pagesize := os.Getpagesize()

	fmt.Println("Environment Info")
	fmt.Println("  Map address:", nil)
	fmt.Println("  Map size:", info.MapSize)
	fmt.Println("  Page size:", pagesize)
	fmt.Println("  Max pages:", info.MapSize/int64(pagesize))
	fmt.Println("  Number of pages used:", info.LastPNO+1)
	fmt.Println("  Last transaction ID:", info.LastTxnID)
	fmt.Println("  Max readers:", info.MaxReaders)
	fmt.Println("  Number of readers used:", info.NumReaders)

	return nil
}

func doPrintReaders(env *lmdb.Env, opt *Options) error {
	fmt.Println("Reader Table Status")
	w := bufio.NewWriter(os.Stdout)
	err := printReaders(env, w, opt)
	if err == nil {
		err = w.Flush()
	}
	if err != nil {
		return err
	}

	if opt.PrintReadersCheck {
		numstale, err := env.ReaderCheck()
		if err != nil {
			return err
		}
		fmt.Printf("  %d stale readers cleared.\n", numstale)
		err = printReaders(env, w, opt)
		if err == nil {
			err = w.Flush()
		}
		if err != nil {
			return err
		}
	}

	return nil
}

func printReaders(env *lmdb.Env, w io.Writer, opt *Options) error {
	return env.ReaderList(func(msg string) error {
		_, err := fmt.Fprint(w, msg)
		return err
	})
}

func doPrintFree(env *lmdb.Env, opt *Options) error {
	return env.View(func(txn *lmdb.Txn) (err error) {
		txn.RawRead = true

		fmt.Println("Freelist Status")

		stat, err := txn.Stat(0)
		if err != nil {
			return err
		}
		printStat(stat, opt)

		var numpages int64
		s := lmdbscan.New(txn, 0)
		defer s.Close()
		for s.Scan() {
			key := s.Key()
			data := s.Val()
			txid := *(*C.size_t)(unsafe.Pointer(&key[0]))
			ipages := int64(*(*C.size_t)(unsafe.Pointer(&data[0])))
			numpages += ipages
			if opt.PrintFreeSummary || opt.PrintFreeFull {
				bad := ""
				hdr := reflect.SliceHeader{
					Data: uintptr(unsafe.Pointer(&data[0])),
					Len:  int(ipages) + 1,
					Cap:  int(ipages) + 1,
				}
				pages := *(*[]C.size_t)(unsafe.Pointer(&hdr))
				pages = pages[1:]
				var span C.size_t
				prev := C.size_t(1)
				for i := ipages - 1; i >= 0; i-- {
					pg := pages[i]
					if pg < prev {
						bad = " [bad sequence]"
					}
					prev = pg
					pg += span
					for i >= int64(span) && pages[i-int64(span)] == pg {
						span++
						pg++
					}
				}
				fmt.Printf("    Transaction %d, %d pages, maxspan %d%s\n", txid, ipages, span, bad)

				if opt.PrintFreeFull {
					for j := ipages - 1; j >= 0; {
						pg := pages[j]
						j--
						span := C.size_t(1)
						for j >= 0 && pages[j] == pg+span {
							j--
							span++
						}
						if span > 1 {
							fmt.Printf("     %9d[%d]\n", pg, span)
						} else {
							fmt.Printf("     %9d\n", pg)
						}
					}
				}
			}
		}
		err = s.Err()
		if err != nil {
			return err
		}

		fmt.Println("  Free pages:", numpages)

		return nil
	})
}

func doPrintStatRoot(env *lmdb.Env, opt *Options) error {
	stat, err := env.Stat()
	if err != nil {
		return err
	}

	fmt.Println("Status of Main DB")
	fmt.Println("  Tree depth:", stat.Depth)
	fmt.Println("  Branch pages:", stat.BranchPages)
	fmt.Println("  Leaf pages:", stat.LeafPages)
	fmt.Println("  Overflow pages:", stat.OverflowPages)
	fmt.Println("  Entries:", stat.Entries)

	return nil
}

func doPrintStatDB(env *lmdb.Env, db string, opt *Options) error {
	err := env.View(func(txn *lmdb.Txn) (err error) {
		return printStatDB(env, txn, db, opt)
	})
	if err != nil {
		return fmt.Errorf("%v (%s)", err, db)
	}
	return nil
}

func printStatDB(env *lmdb.Env, txn *lmdb.Txn, db string, opt *Options) error {
	dbi, err := txn.OpenDBI(db, 0)
	if err != nil {
		return err
	}
	defer env.CloseDBI(dbi)

	stat, err := txn.Stat(dbi)
	if err != nil {
		return err
	}

	fmt.Println("Status of", db)
	printStat(stat, opt)

	return err
}

func printStat(stat *lmdb.Stat, opt *Options) error {
	fmt.Println("  Tree depth:", stat.Depth)
	fmt.Println("  Branch pages:", stat.BranchPages)
	fmt.Println("  Leaf pages:", stat.LeafPages)
	fmt.Println("  Overflow pages:", stat.OverflowPages)
	fmt.Println("  Entries:", stat.Entries)

	return nil
}

func doPrintStatAll(env *lmdb.Env, opt *Options) error {
	return env.View(func(txn *lmdb.Txn) (err error) {
		dbi, err := txn.OpenRoot(0)
		if err != nil {
			return err
		}
		defer env.CloseDBI(dbi)

		s := lmdbscan.New(txn, dbi)
		defer s.Close()
		for s.Scan() {
			err = printStatDB(env, txn, string(s.Key()), opt)
			if e, ok := err.(*lmdb.OpError); ok {
				if e.Op == "mdb_dbi_open" {
					continue
				}
			}
			if err != nil {
				return fmt.Errorf("%v (%s)", err, s.Key())
			}
		}
		return s.Err()
	})
}
<|endoftext|>