<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// The watcher package provides an interface for observing changes
// to arbitrary MongoDB documents that are maintained via the
// mgo/txn transaction package.
package watcher

import (
	"fmt"
	"time"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"launchpad.net/tomb"
)

var logger = loggo.GetLogger("juju.state.watcher")

// A Watcher can watch any number of collections and documents for changes.
type Watcher struct {
	tomb tomb.Tomb
	log  *mgo.Collection

	// watches holds the observers managed by Watch/Unwatch.
	watches map[watchKey][]watchInfo

	// current holds the current txn-revno values for all the observed
	// documents known to exist. Documents not observed or deleted are
	// omitted from this map and are considered to have revno -1.
	current map[watchKey]int64

	// needSync is set when a synchronization should take
	// place.
	needSync bool

	// syncEvents and requestEvents contain the events to be
	// dispatched to the watcher channels. They're queued during
	// processing and flushed at the end to simplify the algorithm.
	// The two queues are separated because events from sync are
	// handled in reverse order due to the way the algorithm works.
	syncEvents, requestEvents []event

	// request is used to deliver requests from the public API into
	// the the goroutine loop.
	request chan interface{}

	// lastId is the most recent transaction id observed by a sync.
	lastId interface{}
}

// A Change holds information about a document change.
type Change struct {
	// C and Id hold the collection name and document _id field value.
	C  string
	Id interface{}

	// Revno is the latest known value for the document's txn-revno
	// field, or -1 if the document was deleted.
	Revno int64
}

type watchKey struct {
	c  string
	id interface{} // nil when watching collection
}

func (k watchKey) String() string {
	coll := "collection " + k.c
	if k.id == nil {
		return coll
	}
	return fmt.Sprintf("document %v in %s", k.id, coll)
}

// match returns whether the receiving watch key,
// which may refer to a particular item or
// an entire collection, matches k1, which refers
// to a particular item.
func (k watchKey) match(k1 watchKey) bool {
	if k.c != k1.c {
		return false
	}
	if k.id == nil {
		// k refers to entire collection
		return true
	}
	return k.id == k1.id
}

type watchInfo struct {
	ch     chan<- Change
	revno  int64
	filter func(interface{}) bool
}

type event struct {
	ch    chan<- Change
	key   watchKey
	revno int64
}

// New returns a new Watcher observing the changelog collection,
// which must be a capped collection maintained by mgo/txn.
func New(changelog *mgo.Collection) *Watcher {
	w := &Watcher{
		log:     changelog,
		watches: make(map[watchKey][]watchInfo),
		current: make(map[watchKey]int64),
		request: make(chan interface{}),
	}
	go func() {
		w.tomb.Kill(w.loop())
		w.tomb.Done()
	}()
	return w
}

// Stop stops all the watcher activities.
func (w *Watcher) Stop() error {
	w.tomb.Kill(nil)
	return w.tomb.Wait()
}

// Dead returns a channel that is closed when the watcher has stopped.
func (w *Watcher) Dead() <-chan struct{} {
	return w.tomb.Dead()
}

// Err returns the error with which the watcher stopped.
// It returns nil if the watcher stopped cleanly, tomb.ErrStillAlive
// if the watcher is still running properly, or the respective error
// if the watcher is terminating or has terminated with an error.
func (w *Watcher) Err() error {
	return w.tomb.Err()
}

type reqWatch struct {
	key  watchKey
	info watchInfo
}

type reqUnwatch struct {
	key watchKey
	ch  chan<- Change
}

type reqSync struct{}

func (w *Watcher) sendReq(req interface{}) {
	select {
	case w.request <- req:
	case <-w.tomb.Dying():
	}
}

// Watch starts watching the given collection and document id.
// An event will be sent onto ch whenever a matching document's txn-revno
// field is observed to change after a transaction is applied. The revno
// parameter holds the currently known revision number for the document.
// Non-existent documents are represented by a -1 revno.
func (w *Watcher) Watch(collection string, id interface{}, revno int64, ch chan<- Change) {
	if id == nil {
		panic("watcher: cannot watch a document with nil id")
	}
	w.sendReq(reqWatch{watchKey{collection, id}, watchInfo{ch, revno, nil}})
}

// WatchCollection starts watching the given collection.
// An event will be sent onto ch whenever the txn-revno field is observed
// to change after a transaction is applied for any document in the collection.
func (w *Watcher) WatchCollection(collection string, ch chan<- Change) {
	w.WatchCollectionWithFilter(collection, ch, nil)
}

// WatchCollectionWithFilter starts watching the given collection.
// An event will be sent onto ch whenever the txn-revno field is observed
// to change after a transaction is applied for any document in the collection, so long as the
// specified filter function returns true when called with the document id value.
func (w *Watcher) WatchCollectionWithFilter(collection string, ch chan<- Change, filter func(interface{}) bool) {
	w.sendReq(reqWatch{watchKey{collection, nil}, watchInfo{ch, 0, filter}})
}

// Unwatch stops watching the given collection and document id via ch.
func (w *Watcher) Unwatch(collection string, id interface{}, ch chan<- Change) {
	if id == nil {
		panic("watcher: cannot unwatch a document with nil id")
	}
	w.sendReq(reqUnwatch{watchKey{collection, id}, ch})
}

// UnwatchCollection stops watching the given collection via ch.
func (w *Watcher) UnwatchCollection(collection string, ch chan<- Change) {
	w.sendReq(reqUnwatch{watchKey{collection, nil}, ch})
}

// StartSync forces the watcher to load new events from the database.
func (w *Watcher) StartSync() {
	w.sendReq(reqSync{})
}

// Period is the delay between each sync.
// It must not be changed when any watchers are active.
var Period time.Duration = 5 * time.Second

// loop implements the main watcher loop.
func (w *Watcher) loop() error {
	next := time.After(Period)
	w.needSync = true
	if err := w.initLastId(); err != nil {
		return err
	}
	for {
		if w.needSync {
			if err := w.sync(); err != nil {
				return err
			}
			w.flush()
			next = time.After(Period)
		}
		select {
		case <-w.tomb.Dying():
			return tomb.ErrDying
		case <-next:
			next = time.After(Period)
			w.needSync = true
		case req := <-w.request:
			w.handle(req)
			w.flush()
		}
	}
}

// flush sends all pending events to their respective channels.
func (w *Watcher) flush() {
	// refreshEvents are stored newest first.
	for i := len(w.syncEvents) - 1; i >= 0; i-- {
		e := &w.syncEvents[i]
		for e.ch != nil {
			select {
			case <-w.tomb.Dying():
				return
			case req := <-w.request:
				w.handle(req)
				continue
			case e.ch <- Change{e.key.c, e.key.id, e.revno}:
			}
			break
		}
	}
	// requestEvents are stored oldest first, and
	// may grow during the loop.
	for i := 0; i < len(w.requestEvents); i++ {
		e := &w.requestEvents[i]
		for e.ch != nil {
			select {
			case <-w.tomb.Dying():
				return
			case req := <-w.request:
				w.handle(req)
				continue
			case e.ch <- Change{e.key.c, e.key.id, e.revno}:
			}
			break
		}
	}
	w.syncEvents = w.syncEvents[:0]
	w.requestEvents = w.requestEvents[:0]
}

// handle deals with requests delivered by the public API
// onto the background watcher goroutine.
func (w *Watcher) handle(req interface{}) {
	logger.Tracef("got request: %#v", req)
	switch r := req.(type) {
	case reqSync:
		w.needSync = true
	case reqWatch:
		for _, info := range w.watches[r.key] {
			if info.ch == r.info.ch {
				panic(fmt.Errorf("tried to re-add channel %v for %s", info.ch, r.key))
			}
		}
		if revno, ok := w.current[r.key]; ok && (revno > r.info.revno || revno == -1 && r.info.revno >= 0) {
			r.info.revno = revno
			w.requestEvents = append(w.requestEvents, event{r.info.ch, r.key, revno})
		}
		w.watches[r.key] = append(w.watches[r.key], r.info)
	case reqUnwatch:
		watches := w.watches[r.key]
		removed := false
		for i, info := range watches {
			if info.ch == r.ch {
				watches[i] = watches[len(watches)-1]
				w.watches[r.key] = watches[:len(watches)-1]
				removed = true
				break
			}
		}
		if !removed {
			panic(fmt.Errorf("tried to remove missing channel %v for %s", r.ch, r.key))
		}
		for i := range w.requestEvents {
			e := &w.requestEvents[i]
			if r.key.match(e.key) && e.ch == r.ch {
				e.ch = nil
			}
		}
		for i := range w.syncEvents {
			e := &w.syncEvents[i]
			if r.key.match(e.key) && e.ch == r.ch {
				e.ch = nil
			}
		}
	default:
		panic(fmt.Errorf("unknown request: %T", req))
	}
}

type logInfo struct {
	Docs   []interface{} `bson:"d"`
	Revnos []int64       `bson:"r"`
}

// initLastId reads the most recent changelog document and initializes
// lastId with it. This causes all history that precedes the creation
// of the watcher to be ignored.
func (w *Watcher) initLastId() error {
	var entry struct {
		Id interface{} `bson:"_id"`
	}
	err := w.log.Find(nil).Sort("-$natural").One(&entry)
	if err != nil && err != mgo.ErrNotFound {
		return err
	}
	w.lastId = entry.Id
	return nil
}

// sync updates the watcher knowledge from the database, and
// queues events to observing channels.
func (w *Watcher) sync() error {
	w.needSync = false
	// Iterate through log events in reverse insertion order (newest first).
	iter := w.log.Find(nil).Batch(10).Sort("-$natural").Iter()
	seen := make(map[watchKey]bool)
	first := true
	lastId := w.lastId
	var entry bson.D
	for iter.Next(&entry) {
		if len(entry) == 0 {
			logger.Tracef("got empty changelog document")
		}
		id := entry[0]
		if id.Name != "_id" {
			panic("watcher: _id field isn't first entry")
		}
		if first {
			w.lastId = id.Value
			first = false
		}
		if id.Value == lastId {
			break
		}
		logger.Tracef("got changelog document: %#v", entry)
		for _, c := range entry[1:] {
			// See txn's Runner.ChangeLog for the structure of log entries.
			var d, r []interface{}
			dr, _ := c.Value.(bson.D)
			for _, item := range dr {
				switch item.Name {
				case "d":
					d, _ = item.Value.([]interface{})
				case "r":
					r, _ = item.Value.([]interface{})
				}
			}
			if len(d) == 0 || len(d) != len(r) {
				logger.Warningf("changelog has invalid collection document: %#v", c)
				continue
			}
			for i := len(d) - 1; i >= 0; i-- {
				key := watchKey{c.Name, d[i]}
				if seen[key] {
					continue
				}
				seen[key] = true
				revno, ok := r[i].(int64)
				if !ok {
					logger.Warningf("changelog has revno with type %T: %#v", r[i], r[i])
					continue
				}
				if revno < 0 {
					revno = -1
				}
				if w.current[key] == revno {
					continue
				}
				w.current[key] = revno
				// Queue notifications for per-collection watches.
				for _, info := range w.watches[watchKey{c.Name, nil}] {
					if info.filter != nil && !info.filter(d[i]) {
						continue
					}
					w.syncEvents = append(w.syncEvents, event{info.ch, key, revno})
				}
				// Queue notifications for per-document watches.
				infos := w.watches[key]
				for i, info := range infos {
					if revno > info.revno || revno < 0 && info.revno >= 0 {
						infos[i].revno = revno
						w.syncEvents = append(w.syncEvents, event{info.ch, key, revno})
					}
				}
			}
		}
	}
	if err := iter.Close(); err != nil {
		return errors.Errorf("watcher iteration error: %v", err)
	}
	return nil
}
<commit_msg>Found another place where collection needs to copy session<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// The watcher package provides an interface for observing changes
// to arbitrary MongoDB documents that are maintained via the
// mgo/txn transaction package.
package watcher

import (
	"fmt"
	"time"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"launchpad.net/tomb"
)

var logger = loggo.GetLogger("juju.state.watcher")

// A Watcher can watch any number of collections and documents for changes.
type Watcher struct {
	tomb tomb.Tomb
	log  *mgo.Collection

	// watches holds the observers managed by Watch/Unwatch.
	watches map[watchKey][]watchInfo

	// current holds the current txn-revno values for all the observed
	// documents known to exist. Documents not observed or deleted are
	// omitted from this map and are considered to have revno -1.
	current map[watchKey]int64

	// needSync is set when a synchronization should take
	// place.
	needSync bool

	// syncEvents and requestEvents contain the events to be
	// dispatched to the watcher channels. They're queued during
	// processing and flushed at the end to simplify the algorithm.
	// The two queues are separated because events from sync are
	// handled in reverse order due to the way the algorithm works.
	syncEvents, requestEvents []event

	// request is used to deliver requests from the public API into
	// the the goroutine loop.
	request chan interface{}

	// lastId is the most recent transaction id observed by a sync.
	lastId interface{}
}

// A Change holds information about a document change.
type Change struct {
	// C and Id hold the collection name and document _id field value.
	C  string
	Id interface{}

	// Revno is the latest known value for the document's txn-revno
	// field, or -1 if the document was deleted.
	Revno int64
}

type watchKey struct {
	c  string
	id interface{} // nil when watching collection
}

func (k watchKey) String() string {
	coll := "collection " + k.c
	if k.id == nil {
		return coll
	}
	return fmt.Sprintf("document %v in %s", k.id, coll)
}

// match returns whether the receiving watch key,
// which may refer to a particular item or
// an entire collection, matches k1, which refers
// to a particular item.
func (k watchKey) match(k1 watchKey) bool {
	if k.c != k1.c {
		return false
	}
	if k.id == nil {
		// k refers to entire collection
		return true
	}
	return k.id == k1.id
}

type watchInfo struct {
	ch     chan<- Change
	revno  int64
	filter func(interface{}) bool
}

type event struct {
	ch    chan<- Change
	key   watchKey
	revno int64
}

// New returns a new Watcher observing the changelog collection,
// which must be a capped collection maintained by mgo/txn.
func New(changelog *mgo.Collection) *Watcher {
	w := &Watcher{
		log:     changelog,
		watches: make(map[watchKey][]watchInfo),
		current: make(map[watchKey]int64),
		request: make(chan interface{}),
	}
	go func() {
		w.tomb.Kill(w.loop())
		w.tomb.Done()
	}()
	return w
}

// Stop stops all the watcher activities.
func (w *Watcher) Stop() error {
	w.tomb.Kill(nil)
	return w.tomb.Wait()
}

// Dead returns a channel that is closed when the watcher has stopped.
func (w *Watcher) Dead() <-chan struct{} {
	return w.tomb.Dead()
}

// Err returns the error with which the watcher stopped.
// It returns nil if the watcher stopped cleanly, tomb.ErrStillAlive
// if the watcher is still running properly, or the respective error
// if the watcher is terminating or has terminated with an error.
func (w *Watcher) Err() error {
	return w.tomb.Err()
}

type reqWatch struct {
	key  watchKey
	info watchInfo
}

type reqUnwatch struct {
	key watchKey
	ch  chan<- Change
}

type reqSync struct{}

func (w *Watcher) sendReq(req interface{}) {
	select {
	case w.request <- req:
	case <-w.tomb.Dying():
	}
}

// Watch starts watching the given collection and document id.
// An event will be sent onto ch whenever a matching document's txn-revno
// field is observed to change after a transaction is applied. The revno
// parameter holds the currently known revision number for the document.
// Non-existent documents are represented by a -1 revno.
func (w *Watcher) Watch(collection string, id interface{}, revno int64, ch chan<- Change) {
	if id == nil {
		panic("watcher: cannot watch a document with nil id")
	}
	w.sendReq(reqWatch{watchKey{collection, id}, watchInfo{ch, revno, nil}})
}

// WatchCollection starts watching the given collection.
// An event will be sent onto ch whenever the txn-revno field is observed
// to change after a transaction is applied for any document in the collection.
func (w *Watcher) WatchCollection(collection string, ch chan<- Change) {
	w.WatchCollectionWithFilter(collection, ch, nil)
}

// WatchCollectionWithFilter starts watching the given collection.
// An event will be sent onto ch whenever the txn-revno field is observed
// to change after a transaction is applied for any document in the collection, so long as the
// specified filter function returns true when called with the document id value.
func (w *Watcher) WatchCollectionWithFilter(collection string, ch chan<- Change, filter func(interface{}) bool) {
	w.sendReq(reqWatch{watchKey{collection, nil}, watchInfo{ch, 0, filter}})
}

// Unwatch stops watching the given collection and document id via ch.
func (w *Watcher) Unwatch(collection string, id interface{}, ch chan<- Change) {
	if id == nil {
		panic("watcher: cannot unwatch a document with nil id")
	}
	w.sendReq(reqUnwatch{watchKey{collection, id}, ch})
}

// UnwatchCollection stops watching the given collection via ch.
func (w *Watcher) UnwatchCollection(collection string, ch chan<- Change) {
	w.sendReq(reqUnwatch{watchKey{collection, nil}, ch})
}

// StartSync forces the watcher to load new events from the database.
func (w *Watcher) StartSync() {
	w.sendReq(reqSync{})
}

// Period is the delay between each sync.
// It must not be changed when any watchers are active.
var Period time.Duration = 5 * time.Second

// loop implements the main watcher loop.
func (w *Watcher) loop() error {
	next := time.After(Period)
	w.needSync = true
	if err := w.initLastId(); err != nil {
		return err
	}
	for {
		if w.needSync {
			if err := w.sync(); err != nil {
				return err
			}
			w.flush()
			next = time.After(Period)
		}
		select {
		case <-w.tomb.Dying():
			return tomb.ErrDying
		case <-next:
			next = time.After(Period)
			w.needSync = true
		case req := <-w.request:
			w.handle(req)
			w.flush()
		}
	}
}

// flush sends all pending events to their respective channels.
func (w *Watcher) flush() {
	// refreshEvents are stored newest first.
	for i := len(w.syncEvents) - 1; i >= 0; i-- {
		e := &w.syncEvents[i]
		for e.ch != nil {
			select {
			case <-w.tomb.Dying():
				return
			case req := <-w.request:
				w.handle(req)
				continue
			case e.ch <- Change{e.key.c, e.key.id, e.revno}:
			}
			break
		}
	}
	// requestEvents are stored oldest first, and
	// may grow during the loop.
	for i := 0; i < len(w.requestEvents); i++ {
		e := &w.requestEvents[i]
		for e.ch != nil {
			select {
			case <-w.tomb.Dying():
				return
			case req := <-w.request:
				w.handle(req)
				continue
			case e.ch <- Change{e.key.c, e.key.id, e.revno}:
			}
			break
		}
	}
	w.syncEvents = w.syncEvents[:0]
	w.requestEvents = w.requestEvents[:0]
}

// handle deals with requests delivered by the public API
// onto the background watcher goroutine.
func (w *Watcher) handle(req interface{}) {
	logger.Tracef("got request: %#v", req)
	switch r := req.(type) {
	case reqSync:
		w.needSync = true
	case reqWatch:
		for _, info := range w.watches[r.key] {
			if info.ch == r.info.ch {
				panic(fmt.Errorf("tried to re-add channel %v for %s", info.ch, r.key))
			}
		}
		if revno, ok := w.current[r.key]; ok && (revno > r.info.revno || revno == -1 && r.info.revno >= 0) {
			r.info.revno = revno
			w.requestEvents = append(w.requestEvents, event{r.info.ch, r.key, revno})
		}
		w.watches[r.key] = append(w.watches[r.key], r.info)
	case reqUnwatch:
		watches := w.watches[r.key]
		removed := false
		for i, info := range watches {
			if info.ch == r.ch {
				watches[i] = watches[len(watches)-1]
				w.watches[r.key] = watches[:len(watches)-1]
				removed = true
				break
			}
		}
		if !removed {
			panic(fmt.Errorf("tried to remove missing channel %v for %s", r.ch, r.key))
		}
		for i := range w.requestEvents {
			e := &w.requestEvents[i]
			if r.key.match(e.key) && e.ch == r.ch {
				e.ch = nil
			}
		}
		for i := range w.syncEvents {
			e := &w.syncEvents[i]
			if r.key.match(e.key) && e.ch == r.ch {
				e.ch = nil
			}
		}
	default:
		panic(fmt.Errorf("unknown request: %T", req))
	}
}

type logInfo struct {
	Docs   []interface{} `bson:"d"`
	Revnos []int64       `bson:"r"`
}

// initLastId reads the most recent changelog document and initializes
// lastId with it. This causes all history that precedes the creation
// of the watcher to be ignored.
func (w *Watcher) initLastId() error {
	var entry struct {
		Id interface{} `bson:"_id"`
	}
	session := w.log.Database.Session.Copy()
	defer session.Close()
	err := w.log.With(session).Find(nil).Sort("-$natural").One(&entry)
	if err != nil && err != mgo.ErrNotFound {
		return err
	}
	w.lastId = entry.Id
	return nil
}

// sync updates the watcher knowledge from the database, and
// queues events to observing channels.
func (w *Watcher) sync() error {
	w.needSync = false
	// Iterate through log events in reverse insertion order (newest first).
	session := w.log.Database.Session.Copy()
	defer session.Close()
	iter := w.log.With(session).Find(nil).Batch(10).Sort("-$natural").Iter()
	seen := make(map[watchKey]bool)
	first := true
	lastId := w.lastId
	var entry bson.D
	for iter.Next(&entry) {
		if len(entry) == 0 {
			logger.Tracef("got empty changelog document")
		}
		id := entry[0]
		if id.Name != "_id" {
			panic("watcher: _id field isn't first entry")
		}
		if first {
			w.lastId = id.Value
			first = false
		}
		if id.Value == lastId {
			break
		}
		logger.Tracef("got changelog document: %#v", entry)
		for _, c := range entry[1:] {
			// See txn's Runner.ChangeLog for the structure of log entries.
			var d, r []interface{}
			dr, _ := c.Value.(bson.D)
			for _, item := range dr {
				switch item.Name {
				case "d":
					d, _ = item.Value.([]interface{})
				case "r":
					r, _ = item.Value.([]interface{})
				}
			}
			if len(d) == 0 || len(d) != len(r) {
				logger.Warningf("changelog has invalid collection document: %#v", c)
				continue
			}
			for i := len(d) - 1; i >= 0; i-- {
				key := watchKey{c.Name, d[i]}
				if seen[key] {
					continue
				}
				seen[key] = true
				revno, ok := r[i].(int64)
				if !ok {
					logger.Warningf("changelog has revno with type %T: %#v", r[i], r[i])
					continue
				}
				if revno < 0 {
					revno = -1
				}
				if w.current[key] == revno {
					continue
				}
				w.current[key] = revno
				// Queue notifications for per-collection watches.
				for _, info := range w.watches[watchKey{c.Name, nil}] {
					if info.filter != nil && !info.filter(d[i]) {
						continue
					}
					w.syncEvents = append(w.syncEvents, event{info.ch, key, revno})
				}
				// Queue notifications for per-document watches.
				infos := w.watches[key]
				for i, info := range infos {
					if revno > info.revno || revno < 0 && info.revno >= 0 {
						infos[i].revno = revno
						w.syncEvents = append(w.syncEvents, event{info.ch, key, revno})
					}
				}
			}
		}
	}
	if err := iter.Close(); err != nil {
		return errors.Errorf("watcher iteration error: %v", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package stacks

import (
	"regexp"
	"strings"
)

var nameFormat = regexp.MustCompile(`^[a-z0-9\-]{3,128}$`)

// CheckName checks name
func CheckName(name string) (string, error) {
	name = strings.TrimSpace(name)
	if name == "" {
		return "", InvalidName
	}
	if !nameFormat.MatchString(name) {
		return "", InvalidName
	}
	return name, nil
}

// Validate validates Stack
func (f *Stack) Validate() (err error) {
	if f.Name, err = CheckName(f.Name); err != nil {
		return err
	}
	return nil
}
<commit_msg>Accept two letter name for stack deploy<commit_after>package stacks

import (
	"regexp"
	"strings"
)

var nameFormat = regexp.MustCompile(`^[a-z0-9\-]{2,128}$`)

// CheckName checks name
func CheckName(name string) (string, error) {
	name = strings.TrimSpace(name)
	if name == "" {
		return "", InvalidName
	}
	if !nameFormat.MatchString(name) {
		return "", InvalidName
	}
	return name, nil
}

// Validate validates Stack
func (f *Stack) Validate() (err error) {
	if f.Name, err = CheckName(f.Name); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package toolbox

import (
	"fmt"
	"reflect"
	"time"
)

//Zeroable represents object that can call IsZero
type Zeroable interface {
	//IsZero returns true, if value of object was zeroed.
	IsZero() bool
}

//IsInt returns true if input is an int
func IsInt(input interface{}) bool {
	switch input.(type) {
	case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
		return true
	}
	return false
}

//IsFloat returns true if input is a float
func IsFloat(input interface{}) bool {
	switch input.(type) {
	case float32, float64:
		return true
	}
	return false
}

//IsBool returns true if input is a boolean
func IsBool(input interface{}) bool {
	switch input.(type) {
	case bool:
		return true
	}
	return false
}

//IsString returns true if input is a string
func IsString(input interface{}) bool {
	switch input.(type) {
	case string:
		return true
	}
	return false
}

//CanConvertToString checks if  input can be converted to string
func CanConvertToString(input interface{}) bool {
	return reflect.TypeOf(input).AssignableTo(reflect.TypeOf(""))
}

//IsTime returns true if input is a time
func IsTime(input interface{}) bool {
	switch input.(type) {
	case time.Time:
		return true
	case *time.Time:
		return true
	}
	return false
}

//IsMap returns true if input is a map
func IsMap(input interface{}) bool {
	switch input.(type) {
	case map[string]interface{}:
		return true
	}
	candidateType := DereferenceType(reflect.TypeOf(input))
	return candidateType.Kind() == reflect.Map
}

//IsStruct returns true if input is a map
func IsStruct(input interface{}) bool {
	inputType := DereferenceType(input)
	return inputType.Kind() == reflect.Struct
}

//IsSlice returns true if input is a map
func IsSlice(input interface{}) bool {
	switch input.(type) {
	case []interface{}:
		return true
	case []string:
		return true
	}
	candidateType := DereferenceType(reflect.TypeOf(input))
	return candidateType.Kind() == reflect.Slice
}

//IsFunc returns true if input is a funct
func IsFunc(input interface{}) bool {
	candidateType := DereferenceType(reflect.TypeOf(input))
	return candidateType.Kind() == reflect.Func
}

//IsZero returns true if input is a zeroable
func IsZero(input interface{}) bool {
	if zeroable, ok := input.(Zeroable); ok {
		return zeroable.IsZero()
	}
	return false
}

//IsPointer returns true if input is a pointer
func IsPointer(input interface{}) bool {
	if reflectType, ok := input.(reflect.Type); ok {
		return reflectType.Kind() == reflect.Ptr
	}
	return reflect.TypeOf(input).Kind() == reflect.Ptr
}

//AssertPointerKind checks if input is a pointer of the passed in kind, if not it panic with message including name
func AssertPointerKind(input interface{}, kind reflect.Kind, name string) {
	AssertTypeKind(reflect.TypeOf(input), reflect.Ptr, name)
	AssertTypeKind(reflect.TypeOf(input).Elem(), kind, name)
}

//AssertKind checks if input is of the passed in kind, if not it panic with message including name
func AssertKind(input interface{}, kind reflect.Kind, name string) {
	AssertTypeKind(reflect.TypeOf(input), kind, name)
}

//AssertTypeKind checks if dataType is of the passed in kind, if not it panic with message including name
func AssertTypeKind(dataType reflect.Type, kind reflect.Kind, name string) {
	if dataType.Kind() != kind {
		panic(fmt.Sprintf("failed to check: %v - expected kind: %v but found %v (%v)", name, kind.String(), dataType.Kind(), dataType.String()))
	}
}

//DiscoverValueByKind returns unwrapped input that matches expected kind, or panic if this is not possible
func DiscoverValueByKind(input interface{}, expected reflect.Kind) reflect.Value {
	value, ok := input.(reflect.Value)
	if !ok {
		value = reflect.ValueOf(input)
	}
	if value.Kind() == expected {
		return value
	} else if value.Kind() == reflect.Ptr {
		return DiscoverValueByKind(value.Elem(), expected)
	} else if value.Kind() == reflect.Interface {
		return DiscoverValueByKind(value.Elem(), expected)
	}
	panic(fmt.Sprintf("failed to discover value by kind expected: %v, actual:%v   on %v:", expected.String(), value.Type(), value))
}

//IsValueOfKind returns true if passed in input is of supplied kind.
func IsValueOfKind(input interface{}, kind reflect.Kind) bool {
	value, ok := input.(reflect.Value)
	if !ok {
		value = reflect.ValueOf(input)
	}
	if value.Kind() == kind {
		return true
	} else if value.Kind() == reflect.Ptr {
		return IsValueOfKind(value.Elem(), kind)
	} else if value.Kind() == reflect.Interface {
		return IsValueOfKind(value.Elem(), kind)
	}
	return false
}

//DiscoverTypeByKind returns unwrapped input type that matches expected kind, or panic if this is not possible
func DiscoverTypeByKind(input interface{}, expected reflect.Kind) reflect.Type {
	value, ok := input.(reflect.Type)
	if !ok {
		value = reflect.TypeOf(input)
	}
	if value.Kind() == expected {
		return value
	} else if value.Kind() == reflect.Ptr || value.Kind() == reflect.Slice {
		return DiscoverTypeByKind(value.Elem(), expected)
	}
	panic(fmt.Sprintf("failed to discover type by kind %v, on %v:", expected.String(), value))
}

//DiscoverComponentType returns type unwrapped from pointer, slice or map
func DiscoverComponentType(input interface{}) reflect.Type {
	value, ok := input.(reflect.Type)
	if !ok {
		value = reflect.TypeOf(input)
	}
	if value.Kind() == reflect.Ptr {
		return DiscoverComponentType(value.Elem())
	} else if value.Kind() == reflect.Slice {
		return value.Elem()
	} else if value.Kind() == reflect.Map {
		return value.Elem()
	}
	return value
}
<commit_msg>add nil check for isStruct<commit_after>package toolbox

import (
	"fmt"
	"reflect"
	"time"
)

//Zeroable represents object that can call IsZero
type Zeroable interface {
	//IsZero returns true, if value of object was zeroed.
	IsZero() bool
}

//IsInt returns true if input is an int
func IsInt(input interface{}) bool {
	switch input.(type) {
	case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
		return true
	}
	return false
}

//IsFloat returns true if input is a float
func IsFloat(input interface{}) bool {
	switch input.(type) {
	case float32, float64:
		return true
	}
	return false
}

//IsBool returns true if input is a boolean
func IsBool(input interface{}) bool {
	switch input.(type) {
	case bool:
		return true
	}
	return false
}

//IsString returns true if input is a string
func IsString(input interface{}) bool {
	switch input.(type) {
	case string:
		return true
	}
	return false
}

//CanConvertToString checks if  input can be converted to string
func CanConvertToString(input interface{}) bool {
	return reflect.TypeOf(input).AssignableTo(reflect.TypeOf(""))
}

//IsTime returns true if input is a time
func IsTime(input interface{}) bool {
	switch input.(type) {
	case time.Time:
		return true
	case *time.Time:
		return true
	}
	return false
}

//IsMap returns true if input is a map
func IsMap(input interface{}) bool {
	switch input.(type) {
	case map[string]interface{}:
		return true
	}
	candidateType := DereferenceType(reflect.TypeOf(input))
	return candidateType.Kind() == reflect.Map
}

//IsStruct returns true if input is a map
func IsStruct(input interface{}) bool {
	if input == nil {
		return false
	}
	inputType := DereferenceType(input)
	return inputType.Kind() == reflect.Struct
}

//IsSlice returns true if input is a map
func IsSlice(input interface{}) bool {
	switch input.(type) {
	case []interface{}:
		return true
	case []string:
		return true
	}
	candidateType := DereferenceType(reflect.TypeOf(input))
	return candidateType.Kind() == reflect.Slice
}

//IsFunc returns true if input is a funct
func IsFunc(input interface{}) bool {
	candidateType := DereferenceType(reflect.TypeOf(input))
	return candidateType.Kind() == reflect.Func
}

//IsZero returns true if input is a zeroable
func IsZero(input interface{}) bool {
	if zeroable, ok := input.(Zeroable); ok {
		return zeroable.IsZero()
	}
	return false
}

//IsPointer returns true if input is a pointer
func IsPointer(input interface{}) bool {
	if reflectType, ok := input.(reflect.Type); ok {
		return reflectType.Kind() == reflect.Ptr
	}
	return reflect.TypeOf(input).Kind() == reflect.Ptr
}

//AssertPointerKind checks if input is a pointer of the passed in kind, if not it panic with message including name
func AssertPointerKind(input interface{}, kind reflect.Kind, name string) {
	AssertTypeKind(reflect.TypeOf(input), reflect.Ptr, name)
	AssertTypeKind(reflect.TypeOf(input).Elem(), kind, name)
}

//AssertKind checks if input is of the passed in kind, if not it panic with message including name
func AssertKind(input interface{}, kind reflect.Kind, name string) {
	AssertTypeKind(reflect.TypeOf(input), kind, name)
}

//AssertTypeKind checks if dataType is of the passed in kind, if not it panic with message including name
func AssertTypeKind(dataType reflect.Type, kind reflect.Kind, name string) {
	if dataType.Kind() != kind {
		panic(fmt.Sprintf("failed to check: %v - expected kind: %v but found %v (%v)", name, kind.String(), dataType.Kind(), dataType.String()))
	}
}

//DiscoverValueByKind returns unwrapped input that matches expected kind, or panic if this is not possible
func DiscoverValueByKind(input interface{}, expected reflect.Kind) reflect.Value {
	value, ok := input.(reflect.Value)
	if !ok {
		value = reflect.ValueOf(input)
	}
	if value.Kind() == expected {
		return value
	} else if value.Kind() == reflect.Ptr {
		return DiscoverValueByKind(value.Elem(), expected)
	} else if value.Kind() == reflect.Interface {
		return DiscoverValueByKind(value.Elem(), expected)
	}
	panic(fmt.Sprintf("failed to discover value by kind expected: %v, actual:%v   on %v:", expected.String(), value.Type(), value))
}

//IsValueOfKind returns true if passed in input is of supplied kind.
func IsValueOfKind(input interface{}, kind reflect.Kind) bool {
	value, ok := input.(reflect.Value)
	if !ok {
		value = reflect.ValueOf(input)
	}
	if value.Kind() == kind {
		return true
	} else if value.Kind() == reflect.Ptr {
		return IsValueOfKind(value.Elem(), kind)
	} else if value.Kind() == reflect.Interface {
		return IsValueOfKind(value.Elem(), kind)
	}
	return false
}

//DiscoverTypeByKind returns unwrapped input type that matches expected kind, or panic if this is not possible
func DiscoverTypeByKind(input interface{}, expected reflect.Kind) reflect.Type {
	value, ok := input.(reflect.Type)
	if !ok {
		value = reflect.TypeOf(input)
	}
	if value.Kind() == expected {
		return value
	} else if value.Kind() == reflect.Ptr || value.Kind() == reflect.Slice {
		return DiscoverTypeByKind(value.Elem(), expected)
	}
	panic(fmt.Sprintf("failed to discover type by kind %v, on %v:", expected.String(), value))
}

//DiscoverComponentType returns type unwrapped from pointer, slice or map
func DiscoverComponentType(input interface{}) reflect.Type {
	value, ok := input.(reflect.Type)
	if !ok {
		value = reflect.TypeOf(input)
	}
	if value.Kind() == reflect.Ptr {
		return DiscoverComponentType(value.Elem())
	} else if value.Kind() == reflect.Slice {
		return value.Elem()
	} else if value.Kind() == reflect.Map {
		return value.Elem()
	}
	return value
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package interfacer

import (
	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

//go:generate go run generate/std/main.go generate/std/pkgs.go -o std.go
//go:generate gofmt -w -s std.go

type cache struct {
	loader.Config
}

func typesInit(paths []string) {
	c = &cache{}
	c.AllowErrors = true
	c.TypeChecker.Error = func(e error) {}
	c.TypeChecker.DisableUnusedImportCheck = true
	c.TypeCheckFuncBodies = func(path string) bool {
		if _, e := stdPkgs[path]; e {
			return false
		}
		return true
	}
}

func typesGet(pkgs []*types.Package) {
	for _, pkg := range pkgs {
		path := pkg.Path()
		if _, e := stdPkgs[path]; e {
			continue
		}
		grabExported(pkg.Scope(), path)
		typesGet(pkg.Imports())
	}
}

func grabExported(scope *types.Scope, path string) {
	ifs, funs := FromScope(scope, false)
	for iftype, ifname := range ifs {
		if _, e := ifaces[iftype]; e {
			continue
		}
		ifaces[iftype] = path + "." + ifname
	}
	for ftype, fname := range funs {
		if _, e := funcs[ftype]; e {
			continue
		}
		funcs[ftype] = path + "." + fname
	}
}
<commit_msg>Don't grab names from a package twice<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package interfacer

import (
	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

//go:generate go run generate/std/main.go generate/std/pkgs.go -o std.go
//go:generate gofmt -w -s std.go

type cache struct {
	loader.Config

	grabbed map[string]struct{}
}

func typesInit(paths []string) {
	c = &cache{
		grabbed: make(map[string]struct{}),
	}
	c.AllowErrors = true
	c.TypeChecker.Error = func(e error) {}
	c.TypeChecker.DisableUnusedImportCheck = true
	c.TypeCheckFuncBodies = func(path string) bool {
		if _, e := stdPkgs[path]; e {
			return false
		}
		return true
	}
}

func typesGet(pkgs []*types.Package) {
	for _, pkg := range pkgs {
		path := pkg.Path()
		if _, e := stdPkgs[path]; e {
			continue
		}
		if _, e := c.grabbed[path]; e {
			continue
		}
		c.grabbed[path] = struct{}{}
		grabExported(pkg.Scope(), path)
		typesGet(pkg.Imports())
	}
}

func grabExported(scope *types.Scope, path string) {
	ifs, funs := FromScope(scope, false)
	for iftype, ifname := range ifs {
		if _, e := ifaces[iftype]; e {
			continue
		}
		ifaces[iftype] = path + "." + ifname
	}
	for ftype, fname := range funs {
		if _, e := funcs[ftype]; e {
			continue
		}
		funcs[ftype] = path + "." + fname
	}
}
<|endoftext|>


<commit_before>package storage

import (
	"encoding/binary"
	"io"

	"github.com/anacrolix/missinggo/x"
	"go.etcd.io/bbolt"

	"github.com/anacrolix/torrent/metainfo"
)

type boltDBPiece struct {
	db  *bbolt.DB
	p   metainfo.Piece
	ih  metainfo.Hash
	key [24]byte
}

var (
	_             PieceImpl = (*boltDBPiece)(nil)
	dataBucketKey           = []byte("data")
)

func (me *boltDBPiece) pc() PieceCompletionGetSetter {
	return boltPieceCompletion{me.db}
}

func (me *boltDBPiece) pk() metainfo.PieceKey {
	return metainfo.PieceKey{me.ih, me.p.Index()}
}

func (me *boltDBPiece) Completion() Completion {
	c, err := me.pc().Get(me.pk())
	x.Pie(err)
	return c
}

func (me *boltDBPiece) MarkComplete() error {
	return me.pc().Set(me.pk(), true)
}

func (me *boltDBPiece) MarkNotComplete() error {
	return me.pc().Set(me.pk(), false)
}
func (me *boltDBPiece) ReadAt(b []byte, off int64) (n int, err error) {
	err = me.db.View(func(tx *bbolt.Tx) error {
		db := tx.Bucket(dataBucketKey)
		if db == nil {
			return io.EOF
		}
		ci := off / chunkSize
		off %= chunkSize
		for len(b) != 0 {
			ck := me.chunkKey(int(ci))
			_b := db.Get(ck[:])
			// If the chunk is the wrong size, assume it's missing as we can't rely on the data.
			if len(_b) != chunkSize {
				return io.EOF
			}
			n1 := copy(b, _b[off:])
			off = 0
			ci++
			b = b[n1:]
			n += n1
		}
		return nil
	})
	return
}

func (me *boltDBPiece) chunkKey(index int) (ret [26]byte) {
	copy(ret[:], me.key[:])
	binary.BigEndian.PutUint16(ret[24:], uint16(index))
	return
}

func (me *boltDBPiece) WriteAt(b []byte, off int64) (n int, err error) {
	err = me.db.Update(func(tx *bbolt.Tx) error {
		db, err := tx.CreateBucketIfNotExists(dataBucketKey)
		if err != nil {
			return err
		}
		ci := off / chunkSize
		off %= chunkSize
		for len(b) != 0 {
			_b := make([]byte, chunkSize)
			ck := me.chunkKey(int(ci))
			copy(_b, db.Get(ck[:]))
			n1 := copy(_b[off:], b)
			db.Put(ck[:], _b)
			if n1 > len(b) {
				break
			}
			b = b[n1:]
			off = 0
			ci++
			n += n1
		}
		return nil
	})
	return
}
<commit_msg>Fix panic in when bbolt storage has Closed<commit_after>package storage

import (
	"encoding/binary"
	"io"

	"go.etcd.io/bbolt"

	"github.com/anacrolix/torrent/metainfo"
)

type boltDBPiece struct {
	db  *bbolt.DB
	p   metainfo.Piece
	ih  metainfo.Hash
	key [24]byte
}

var (
	_             PieceImpl = (*boltDBPiece)(nil)
	dataBucketKey           = []byte("data")
)

func (me *boltDBPiece) pc() PieceCompletionGetSetter {
	return boltPieceCompletion{me.db}
}

func (me *boltDBPiece) pk() metainfo.PieceKey {
	return metainfo.PieceKey{me.ih, me.p.Index()}
}

func (me *boltDBPiece) Completion() Completion {
	c, err := me.pc().Get(me.pk())
	switch err {
	case bbolt.ErrDatabaseNotOpen:
		return Completion{}
	case nil:
	default:
		panic(err)
	}
	return c
}

func (me *boltDBPiece) MarkComplete() error {
	return me.pc().Set(me.pk(), true)
}

func (me *boltDBPiece) MarkNotComplete() error {
	return me.pc().Set(me.pk(), false)
}
func (me *boltDBPiece) ReadAt(b []byte, off int64) (n int, err error) {
	err = me.db.View(func(tx *bbolt.Tx) error {
		db := tx.Bucket(dataBucketKey)
		if db == nil {
			return io.EOF
		}
		ci := off / chunkSize
		off %= chunkSize
		for len(b) != 0 {
			ck := me.chunkKey(int(ci))
			_b := db.Get(ck[:])
			// If the chunk is the wrong size, assume it's missing as we can't rely on the data.
			if len(_b) != chunkSize {
				return io.EOF
			}
			n1 := copy(b, _b[off:])
			off = 0
			ci++
			b = b[n1:]
			n += n1
		}
		return nil
	})
	return
}

func (me *boltDBPiece) chunkKey(index int) (ret [26]byte) {
	copy(ret[:], me.key[:])
	binary.BigEndian.PutUint16(ret[24:], uint16(index))
	return
}

func (me *boltDBPiece) WriteAt(b []byte, off int64) (n int, err error) {
	err = me.db.Update(func(tx *bbolt.Tx) error {
		db, err := tx.CreateBucketIfNotExists(dataBucketKey)
		if err != nil {
			return err
		}
		ci := off / chunkSize
		off %= chunkSize
		for len(b) != 0 {
			_b := make([]byte, chunkSize)
			ck := me.chunkKey(int(ci))
			copy(_b, db.Get(ck[:]))
			n1 := copy(_b[off:], b)
			db.Put(ck[:], _b)
			if n1 > len(b) {
				break
			}
			b = b[n1:]
			off = 0
			ci++
			n += n1
		}
		return nil
	})
	return
}
<|endoftext|>


<commit_before>// Copyright 2013 Miek Gieben. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pkcs11

/*
#define CK_PTR *
#ifndef NULL_PTR
#define NULL_PTR 0
#endif
#define CK_DEFINE_FUNCTION(returnType, name) returnType name
#define CK_DECLARE_FUNCTION(returnType, name) returnType name
#define CK_DECLARE_FUNCTION_POINTER(returnType, name) returnType (* name)
#define CK_CALLBACK_FUNCTION(returnType, name) returnType (* name)

#include <stdlib.h>
#include "pkcs11.h"

CK_ULONG Index(CK_ULONG_PTR array, CK_ULONG i)
{
	return array[i];
}

CK_ULONG Sizeof()
{
	return sizeof(CK_ULONG);
}
*/
import "C"

import (
	"fmt"
	"time"
	"unsafe"
)

// toList converts from a C style array to a []uint.
func toList(clist C.CK_ULONG_PTR, size C.CK_ULONG) []uint {
	l := make([]uint, int(size))
	for i := 0; i < len(l); i++ {
		l[i] = uint(C.Index(clist, C.CK_ULONG(i)))
	}
	defer C.free(unsafe.Pointer(clist))
	return l
}

// cBBool converts a bool to a CK_BBOOL.
func cBBool(x bool) C.CK_BBOOL {
	if x {
		return C.CK_BBOOL(C.CK_TRUE)
	}
	return C.CK_BBOOL(C.CK_FALSE)
}

// Error represents an PKCS#11 error.
type Error uint

func (e Error) Error() string {
	return fmt.Sprintf("pkcs11: 0x%X: %s", uint(e), strerror[uint(e)])
}

func toError(e C.CK_RV) error {
	if e == C.CKR_OK {
		return nil
	}
	return Error(e)
}

/* SessionHandle is a Cryptoki-assigned value that identifies a session. */
type SessionHandle uint

/* ObjectHandle is a token-specific identifier for an object.  */
type ObjectHandle uint

// Version represents any version information from the library.
type Version struct {
	Major byte
	Minor byte
}

func toVersion(version C.CK_VERSION) Version {
	return Version{byte(version.major), byte(version.minor)}
}

// SlotEvent holds the SlotID which for which an slot event (token insertion,
// removal, etc.) occurred.
type SlotEvent struct {
	SlotID uint
}

// Info provides information about the library and hardware used.
type Info struct {
	CryptokiVersion    Version
	ManufacturerID     string
	Flags              uint
	LibraryDescription string
	LibraryVersion     Version
}

/* SlotInfo provides information about a slot. */
type SlotInfo struct {
	SlotDescription string // 64 bytes.
	ManufacturerID  string // 32 bytes.
	Flags           uint
	HardwareVersion Version
	FirmwareVersion Version
}

/* TokenInfo provides information about a token. */
type TokenInfo struct {
	Label              string
	ManufacturerID     string
	Model              string
	SerialNumber       string
	Flags              uint
	MaxSessionCount    uint
	SessionCount       uint
	MaxRwSessionCount  uint
	RwSessionCount     uint
	MaxPinLen          uint
	MinPinLen          uint
	TotalPublicMemory  uint
	FreePublicMemory   uint
	TotalPrivateMemory uint
	FreePrivateMemory  uint
	HardwareVersion    Version
	FirmwareVersion    Version
	UTCTime            string
}

/* SesionInfo provides information about a session. */
type SessionInfo struct {
	SlotID      uint
	State       uint
	Flags       uint
	DeviceError uint
}

// Attribute holds an attribute type/value combination.
type Attribute struct {
	Type  uint
	Value []byte
}

// NewAttribute allocates a Attribute and returns a pointer to it.
// Note that this is merely a convience function, as values returned
// from the HSM are not converted back to Go values, those are just raw
// byte slices.
func NewAttribute(typ uint, x interface{}) *Attribute {
	// This function nicely transforms *to* an attribute, but there is
	// no corresponding function that transform back *from* an attribute,
	// which in PKCS#11 is just an byte array.
	a := new(Attribute)
	a.Type = typ
	if x == nil {
		return a
	}
	switch x.(type) {
	case bool: // create bbool
		if x.(bool) {
			a.Value = []byte{1}
			break
		}
		a.Value = []byte{0}
	case uint, int:
		var y uint
		if _, ok := x.(int); ok {
			y = uint(x.(int))
		}
		if _, ok := x.(uint); ok {
			y = x.(uint)
		}
		// TODO(miek): ugly!
		switch int(C.Sizeof()) {
		case 4:
			a.Value = make([]byte, 4)
			a.Value[0] = byte(y)
			a.Value[1] = byte(y >> 8)
			a.Value[2] = byte(y >> 16)
			a.Value[3] = byte(y >> 24)
		case 8:
			a.Value = make([]byte, 8)
			a.Value[0] = byte(y)
			a.Value[1] = byte(y >> 8)
			a.Value[2] = byte(y >> 16)
			a.Value[3] = byte(y >> 24)
			a.Value[4] = byte(y >> 32)
			a.Value[5] = byte(y >> 40)
			a.Value[6] = byte(y >> 48)
			a.Value[7] = byte(y >> 56)
		}
	case string:
		a.Value = []byte(x.(string))
	case []byte: // just copy
		a.Value = x.([]byte)
	case time.Time: // for CKA_DATE
		a.Value = cDate(x.(time.Time))
	default:
		panic("pkcs11: unhandled attribute type")
	}
	return a
}

// cAttribute returns the start address and the length of an attribute list.
func cAttributeList(a []*Attribute) (C.CK_ATTRIBUTE_PTR, C.CK_ULONG) {
	if len(a) == 0 {
		return nil, 0
	}
	pa := make([]C.CK_ATTRIBUTE, len(a))
	for i := 0; i < len(a); i++ {
		pa[i]._type = C.CK_ATTRIBUTE_TYPE(a[i].Type)
		if a[i].Value == nil {
			continue
		}
		pa[i].pValue = C.CK_VOID_PTR((&a[i].Value[0]))
		pa[i].ulValueLen = C.CK_ULONG(len(a[i].Value))
	}
	return C.CK_ATTRIBUTE_PTR(&pa[0]), C.CK_ULONG(len(a))
}

func cDate(t time.Time) []byte {
	b := make([]byte, 8)
	year, month, day := t.Date()
	y := fmt.Sprintf("%4d", year)
	m := fmt.Sprintf("%02d", month)
	d1 := fmt.Sprintf("%02d", day)
	b[0], b[1], b[2], b[3] = y[0], y[1], y[2], y[3]
	b[4], b[5] = m[0], m[1]
	b[6], b[7] = d1[0], d1[1]
	return b
}

// Mechanism holds an mechanism type/value combination.
type Mechanism struct {
	Mechanism uint
	Parameter []byte
}

func NewMechanism(mech uint, x interface{}) *Mechanism {
	m := new(Mechanism)
	m.Mechanism = mech
	if x == nil {
		return m
	}

	// Add any parameters passed (For now presume always bytes were passed in, is there another case?)
	m.Parameter = x.([]byte)

	return m
}

func cMechanismList(m []*Mechanism) (C.CK_MECHANISM_PTR, C.CK_ULONG) {
	if len(m) == 0 {
		return nil, 0
	}
	pm := make([]C.CK_MECHANISM, len(m))
	for i := 0; i < len(m); i++ {
		pm[i].mechanism = C.CK_MECHANISM_TYPE(m[i].Mechanism)
		if m[i].Parameter == nil {
			continue
		}
		pm[i].pParameter = C.CK_VOID_PTR(&(m[i].Parameter[0]))
		pm[i].ulParameterLen = C.CK_ULONG(len(m[i].Parameter))
	}
	return C.CK_MECHANISM_PTR(&pm[0]), C.CK_ULONG(len(m))
}

// MechanismInfo provides information about a particular mechanism.
type MechanismInfo struct {
	MinKeySize uint
	MaxKeySize uint
	Flags      uint
}
<commit_msg>Clean up NewAttribute<commit_after>// Copyright 2013 Miek Gieben. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pkcs11

/*
#define CK_PTR *
#ifndef NULL_PTR
#define NULL_PTR 0
#endif
#define CK_DEFINE_FUNCTION(returnType, name) returnType name
#define CK_DECLARE_FUNCTION(returnType, name) returnType name
#define CK_DECLARE_FUNCTION_POINTER(returnType, name) returnType (* name)
#define CK_CALLBACK_FUNCTION(returnType, name) returnType (* name)

#include <stdlib.h>
#include "pkcs11.h"

CK_ULONG Index(CK_ULONG_PTR array, CK_ULONG i)
{
	return array[i];
}
*/
import "C"

import (
	"fmt"
	"time"
	"unsafe"
)

// toList converts from a C style array to a []uint.
func toList(clist C.CK_ULONG_PTR, size C.CK_ULONG) []uint {
	l := make([]uint, int(size))
	for i := 0; i < len(l); i++ {
		l[i] = uint(C.Index(clist, C.CK_ULONG(i)))
	}
	defer C.free(unsafe.Pointer(clist))
	return l
}

// cBBool converts a bool to a CK_BBOOL.
func cBBool(x bool) C.CK_BBOOL {
	if x {
		return C.CK_BBOOL(C.CK_TRUE)
	}
	return C.CK_BBOOL(C.CK_FALSE)
}

func uintToBytes(x uint64) []byte {
	ul := C.CK_ULONG(x)
	return C.GoBytes(unsafe.Pointer(&ul), C.int(unsafe.Sizeof(ul)))
}

// Error represents an PKCS#11 error.
type Error uint

func (e Error) Error() string {
	return fmt.Sprintf("pkcs11: 0x%X: %s", uint(e), strerror[uint(e)])
}

func toError(e C.CK_RV) error {
	if e == C.CKR_OK {
		return nil
	}
	return Error(e)
}

/* SessionHandle is a Cryptoki-assigned value that identifies a session. */
type SessionHandle uint

/* ObjectHandle is a token-specific identifier for an object.  */
type ObjectHandle uint

// Version represents any version information from the library.
type Version struct {
	Major byte
	Minor byte
}

func toVersion(version C.CK_VERSION) Version {
	return Version{byte(version.major), byte(version.minor)}
}

// SlotEvent holds the SlotID which for which an slot event (token insertion,
// removal, etc.) occurred.
type SlotEvent struct {
	SlotID uint
}

// Info provides information about the library and hardware used.
type Info struct {
	CryptokiVersion    Version
	ManufacturerID     string
	Flags              uint
	LibraryDescription string
	LibraryVersion     Version
}

/* SlotInfo provides information about a slot. */
type SlotInfo struct {
	SlotDescription string // 64 bytes.
	ManufacturerID  string // 32 bytes.
	Flags           uint
	HardwareVersion Version
	FirmwareVersion Version
}

/* TokenInfo provides information about a token. */
type TokenInfo struct {
	Label              string
	ManufacturerID     string
	Model              string
	SerialNumber       string
	Flags              uint
	MaxSessionCount    uint
	SessionCount       uint
	MaxRwSessionCount  uint
	RwSessionCount     uint
	MaxPinLen          uint
	MinPinLen          uint
	TotalPublicMemory  uint
	FreePublicMemory   uint
	TotalPrivateMemory uint
	FreePrivateMemory  uint
	HardwareVersion    Version
	FirmwareVersion    Version
	UTCTime            string
}

/* SesionInfo provides information about a session. */
type SessionInfo struct {
	SlotID      uint
	State       uint
	Flags       uint
	DeviceError uint
}

// Attribute holds an attribute type/value combination.
type Attribute struct {
	Type  uint
	Value []byte
}

// NewAttribute allocates a Attribute and returns a pointer to it.
// Note that this is merely a convience function, as values returned
// from the HSM are not converted back to Go values, those are just raw
// byte slices.
func NewAttribute(typ uint, x interface{}) *Attribute {
	// This function nicely transforms *to* an attribute, but there is
	// no corresponding function that transform back *from* an attribute,
	// which in PKCS#11 is just an byte array.
	a := new(Attribute)
	a.Type = typ
	if x == nil {
		return a
	}
	switch v := x.(type) {
	case bool:
		if v {
			a.Value = []byte{1}
		} else {
			a.Value = []byte{0}
		}
	case int:
		a.Value = uintToBytes(uint64(v))
	case uint:
		a.Value = uintToBytes(uint64(v))
	case string:
		a.Value = []byte(v)
	case []byte:
		a.Value = v
	case time.Time: // for CKA_DATE
		a.Value = cDate(v)
	default:
		panic("pkcs11: unhandled attribute type")
	}
	return a
}

// cAttribute returns the start address and the length of an attribute list.
func cAttributeList(a []*Attribute) (C.CK_ATTRIBUTE_PTR, C.CK_ULONG) {
	if len(a) == 0 {
		return nil, 0
	}
	pa := make([]C.CK_ATTRIBUTE, len(a))
	for i := 0; i < len(a); i++ {
		pa[i]._type = C.CK_ATTRIBUTE_TYPE(a[i].Type)
		if a[i].Value == nil {
			continue
		}
		pa[i].pValue = C.CK_VOID_PTR((&a[i].Value[0]))
		pa[i].ulValueLen = C.CK_ULONG(len(a[i].Value))
	}
	return C.CK_ATTRIBUTE_PTR(&pa[0]), C.CK_ULONG(len(a))
}

func cDate(t time.Time) []byte {
	b := make([]byte, 8)
	year, month, day := t.Date()
	y := fmt.Sprintf("%4d", year)
	m := fmt.Sprintf("%02d", month)
	d1 := fmt.Sprintf("%02d", day)
	b[0], b[1], b[2], b[3] = y[0], y[1], y[2], y[3]
	b[4], b[5] = m[0], m[1]
	b[6], b[7] = d1[0], d1[1]
	return b
}

// Mechanism holds an mechanism type/value combination.
type Mechanism struct {
	Mechanism uint
	Parameter []byte
}

func NewMechanism(mech uint, x interface{}) *Mechanism {
	m := new(Mechanism)
	m.Mechanism = mech
	if x == nil {
		return m
	}

	// Add any parameters passed (For now presume always bytes were passed in, is there another case?)
	m.Parameter = x.([]byte)

	return m
}

func cMechanismList(m []*Mechanism) (C.CK_MECHANISM_PTR, C.CK_ULONG) {
	if len(m) == 0 {
		return nil, 0
	}
	pm := make([]C.CK_MECHANISM, len(m))
	for i := 0; i < len(m); i++ {
		pm[i].mechanism = C.CK_MECHANISM_TYPE(m[i].Mechanism)
		if m[i].Parameter == nil {
			continue
		}
		pm[i].pParameter = C.CK_VOID_PTR(&(m[i].Parameter[0]))
		pm[i].ulParameterLen = C.CK_ULONG(len(m[i].Parameter))
	}
	return C.CK_MECHANISM_PTR(&pm[0]), C.CK_ULONG(len(m))
}

// MechanismInfo provides information about a particular mechanism.
type MechanismInfo struct {
	MinKeySize uint
	MaxKeySize uint
	Flags      uint
}
<|endoftext|>


<commit_before>package web

//import "gopkg.in/webnice/debug.v1"
//import "gopkg.in/webnice/log.v2"
import "gopkg.in/webnice/web.v1/route"
import "gopkg.in/webnice/web.v1/context/errors"
import "gopkg.in/webnice/web.v1/context/handlers"
import (
	"net"
	"net/http"
	"sync"
	"sync/atomic"
	"time"
)

// Interface is an interface
type Interface interface {
	// ListenAndServe listens on the TCP network address addr and then calls Serve on incoming connections
	ListenAndServe(string) Interface

	// ListenAndServeWithConfig Fully configurable web server listens and then calls Serve on incoming connections
	ListenAndServeWithConfig(*Configuration) Interface

	// Serve accepts incoming connections on the Listener, creating a new service goroutine for each
	Serve(net.Listener) Interface

	// Error Return last error of web server
	Error() error

	// Wait while web server is running
	Wait() Interface

	// Stop web server
	Stop() Interface

	// Route interface
	Route() route.Interface

	// Errors interface
	Errors() errors.Interface

	// Handlers interface
	Handlers() handlers.Interface
}

// Is an private implementation of web server
type web struct {
	isRun       atomic.Value   // The indicator of web server goroutine. =true-goroutine is started, =false-goroutine is stopped
	inCloseUp   chan bool      // The indicator of web server state, true in channel means we're in shutdown goroutine and web server
	doCloseDone sync.WaitGroup // Wait while goroutine stopped

	conf     *Configuration  // The web server configuration
	listener net.Listener    // The web server listener
	server   *http.Server    // The net/http web server object
	route    route.Interface // Routing settings interface
	err      error           // The last of error
}

// Configuration is a structure of web server configuration
type Configuration struct {
	// HostPort (readonly) Адрес составленный автоматически из Host:Port
	// Значение создаётся автоматически при инициализации конфигурации
	// Default value: ":http"
	HostPort string `yaml:"-" json:"-"`

	// Address Публичный адрес на котором сервер доступен извне
	// Например если сервер находится за прокси, тут указывается реальный адрес подключения к серверу
	// Default value: "" - make automatically
	Address string `yaml:"Address" json:"address"`

	// Domains Список всех доменов, на которые отвечает сервер
	// Если не пусто, то для всех других доменов будет ответ "Requested host unavailable"
	// Default value: [] - all domain
	//TODO
	//Domains []string `yaml:"Domains" json:"domains"`

	// Host IP адрес или имя хоста на котором запускается web сервер,
	// можно указывать 0.0.0.0 для всех ip адресов
	// Default value: ""
	Host string `yaml:"Host" json:"host"`

	// Port tcp/ip порт занимаемый сервером
	// Default value: 80
	Port uint32 `yaml:"Port" json:"port"`

	// Socket Unix socket на котором поднимается сервер, только для unix-like операционных систем Linux, Unix, Mac
	// Default value: "" - unix socket is off
	Socket string `yaml:"Socket" json:"socket"`

	// Mode Режим работы, tcp, tcp4, tcp6, unix, unixpacket, socket
	// Default value: "tcp"
	Mode string `yaml:"Mode" json:"mode"`

	// ReadTimeout Время в наносекундах ожидания запроса включая ReadHeaderTimeout
	// Если не указано или рано 0 - таймаута нет
	// Default value: 0 - no timeout
	ReadTimeout time.Duration `yaml:"ReadTimeout" json:"readTimeout"`

	// ReadHeaderTimeout Время в наносекундах ожидания заголовка запроса
	// Если не указано или рано 0 - таймаута нет
	// Default value: 0 - no timeout
	ReadHeaderTimeout time.Duration `yaml:"ReadHeaderTimeout" json:"readHeaderTimeout"`

	// WriteTimeout Время в наносекундах ожидания выдачи ответа
	// Если не указано или рано 0 - таймаута нет
	// Default value: 0 - no timeout
	WriteTimeout time.Duration `yaml:"WriteTimeout" json:"writeTimeout"`

	// IdleTimeout is the maximum amount of time to wait for the next request when keep-alives are enabled
	// Если не указано или рано 0 - таймаута нет
	// Default value: 0 - no timeout
	IdleTimeout time.Duration `yaml:"IdleTimeout" json:"idleTimeout"`

	// ShutdownTimeout is the maximum amount of time to wait for the server graceful shutdown
	// Если не указано или рано 0 - таймаута нет
	// Default value: 30s - One minute
	ShutdownTimeout time.Duration `yaml:"ShutdownTimeout" json:"shutdownTimeout"`

	// MaxHeaderBytes controls the maximum number of bytes the server will read parsing the request header's keys and values, including the request line
	// Default value: 1 MB (from net/http/DefaultMaxHeaderBytes)
	MaxHeaderBytes int `yaml:"MaxHeaderBytes" json:"maxHeaderBytes"`

	// MaxBodyBytes controls the maximum number of bytes the server will read request body
	// Default value: 0 - no limit
	//TODO
	//MaxBodyBytes int64 `yaml:"MaxBodyBytes" json:"maxBodyBytes"`

	// KeepAliveDisable if is equal true, keep alive are disabled, if false - keep alive are enabled
	// Default value: false - keep alive are enabled
	KeepAliveDisable bool `yaml:"KeepAliveDisable" json:"keepAliveDisable"`
}
<commit_msg>Updated comments<commit_after>package web

//import "gopkg.in/webnice/debug.v1"
//import "gopkg.in/webnice/log.v2"
import "gopkg.in/webnice/web.v1/route"
import "gopkg.in/webnice/web.v1/context/errors"
import "gopkg.in/webnice/web.v1/context/handlers"
import (
	"net"
	"net/http"
	"sync"
	"sync/atomic"
	"time"
)

// Interface is an interface
type Interface interface {
	// ListenAndServe listens on the TCP network address addr and then calls Serve on incoming connections
	ListenAndServe(string) Interface

	// ListenAndServeWithConfig Fully configurable web server listens and then calls Serve on incoming connections
	ListenAndServeWithConfig(*Configuration) Interface

	// Serve accepts incoming connections on the Listener, creating a new service goroutine for each
	Serve(net.Listener) Interface

	// Error Return last error of web server
	Error() error

	// Wait while web server is running
	Wait() Interface

	// Stop web server
	Stop() Interface

	// Route interface
	Route() route.Interface

	// Errors interface
	Errors() errors.Interface

	// Handlers interface
	Handlers() handlers.Interface
}

// Is an private implementation of web server
type web struct {
	isRun       atomic.Value   // The indicator of web server goroutine. =true-goroutine is started, =false-goroutine is stopped
	inCloseUp   chan bool      // The indicator of web server state, true in channel means we're in shutdown goroutine and web server
	doCloseDone sync.WaitGroup // Wait while goroutine stopped

	conf     *Configuration  // The web server configuration
	listener net.Listener    // The web server listener
	server   *http.Server    // The net/http web server object
	route    route.Interface // Routing settings interface
	err      error           // The last of error
}

// Configuration is a structure of web server configuration
type Configuration struct {
	// HostPort (readonly) Адрес составленный автоматически из Host:Port
	// Значение создаётся автоматически при инициализации конфигурации
	// Default value: ":http"
	HostPort string `yaml:"-" json:"-"`

	// Address Публичный адрес на котором сервер доступен извне
	// Например если сервер находится за прокси, тут указывается реальный адрес подключения к серверу
	// Default value: "" - make automatically
	Address string `yaml:"Address" json:"address"`

	// Domains Список всех доменов, на которые отвечает сервер
	// Если не пусто, то для всех других доменов будет ответ "Requested host unavailable"
	// Default value: [] - all domain
	//TODO
	//Domains []string `yaml:"Domains" json:"domains"`

	// Host IP адрес или имя хоста на котором запускается web сервер,
	// можно указывать 0.0.0.0 для всех ip адресов
	// Default value: ""
	Host string `yaml:"Host" json:"host"`

	// Port tcp/ip порт занимаемый сервером
	// Default value: 80
	Port uint32 `yaml:"Port" json:"port"`

	// Socket Unix socket на котором поднимается сервер, только для unix-like операционных систем Linux, Unix, Mac
	// Default value: "" - unix socket is off
	Socket string `yaml:"Socket" json:"socket"`

	// Mode Режим работы, tcp, tcp4, tcp6, unix, unixpacket, socket
	// Default value: "tcp"
	Mode string `yaml:"Mode" json:"mode"`

	// ReadTimeout Время в наносекундах ожидания запроса включая ReadHeaderTimeout
	// Если не указано или рано 0 - таймаута нет
	// Default value: 0 - no timeout
	ReadTimeout time.Duration `yaml:"ReadTimeout" json:"readTimeout"`

	// ReadHeaderTimeout Время в наносекундах ожидания заголовка запроса
	// Если не указано или рано 0 - таймаута нет
	// Default value: 0 - no timeout
	ReadHeaderTimeout time.Duration `yaml:"ReadHeaderTimeout" json:"readHeaderTimeout"`

	// WriteTimeout Время в наносекундах ожидания выдачи ответа
	// Если не указано или рано 0 - таймаута нет
	// Default value: 0 - no timeout
	WriteTimeout time.Duration `yaml:"WriteTimeout" json:"writeTimeout"`

	// IdleTimeout is the maximum amount of time to wait for the next request when keep-alives are enabled
	// Если не указано или рано 0 - таймаута нет
	// Default value: 0 - no timeout
	IdleTimeout time.Duration `yaml:"IdleTimeout" json:"idleTimeout"`

	// ShutdownTimeout is the maximum amount of time to wait for the server graceful shutdown
	// Если не указано или рано 0 - таймаута нет
	// Default value: 30s
	ShutdownTimeout time.Duration `yaml:"ShutdownTimeout" json:"shutdownTimeout"`

	// MaxHeaderBytes controls the maximum number of bytes the server will read parsing the request header's keys and values, including the request line
	// Default value: 1 MB (from net/http/DefaultMaxHeaderBytes)
	MaxHeaderBytes int `yaml:"MaxHeaderBytes" json:"maxHeaderBytes"`

	// MaxBodyBytes controls the maximum number of bytes the server will read request body
	// Default value: 0 - unlimited
	//TODO
	//MaxBodyBytes int64 `yaml:"MaxBodyBytes" json:"maxBodyBytes"`

	// KeepAliveDisable if is equal true, keep alive are disabled, if false - keep alive are enabled
	// Default value: false - keep alive are enabled
	KeepAliveDisable bool `yaml:"KeepAliveDisable" json:"keepAliveDisable"`
}
<|endoftext|>


<commit_before>package bla

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"text/template"
	"time"

	"golang.org/x/net/webdav"

	"github.com/fsnotify/fsnotify"
	"github.com/goji/httpauth"
)

// Initialize
type Handler struct {
	cfgPath string
	Cfg     *Config
	public  http.Handler
	webfs   http.Handler
	tpl     *template.Template

	publicPath   string
	templatePath string
	docPath      string

	mu       sync.RWMutex
	docs     map[string]*Doc
	sortDocs []*Doc
	tags     map[string][]*Doc
}

func NewHandler(cfgPath string) *Handler {

	h := &Handler{
		cfgPath: cfgPath,
		mu:      sync.RWMutex{},
	}

	h.loadConfig()
	h.watch()
	h.loadWebDav()

	return h
}

func (s *Handler) loadWebDav() {

	fs := webdav.Dir(s.Cfg.RootPath)
	ls := webdav.NewMemLS()

	handler := &webdav.Handler{
		Prefix:     "/fs",
		FileSystem: fs,
		LockSystem: ls,
	}
	th := throttled.Interval(throttled.PerSec(10),
		1, &throttled.VaryBy{Path: true}, 50)
	wrapper := httpauth.SimpleBasicAuth("admin", s.Cfg.Password)

	s.webfs = th.Throttle(wrapper(handler))
}

func (s *Handler) watch() {

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	go func() {
		// loadData minial interval is 1 second
		ticker := time.NewTicker(500 * time.Millisecond)
		mod := true
		rootChange := true

		for {
			select {
			case event := <-watcher.Events:
				switch ext := filepath.Ext(event.Name); ext {
				case ".md", ".json", ".tmpl":
					log.Println("modified file:", event.Name)
					mod = true
				case ".swp":
					continue
				}

				rootChange = true
			case err := <-watcher.Errors:
				log.Println("error:", err)
				return
			case <-ticker.C:
				if mod {
					mod = false
					s.loadData()
					s.loadTemplate()
				}

				if rootChange {
					rootChange = false
					err := s.saveAll()
					if err != nil {
						log.Fatal("can't save docs:", err)
					}
				}
			}
		}
	}()

	watcher.Add(s.Cfg.RootPath)
	watcher.Add(s.docPath)
	watcher.Add(s.templatePath)

	if err != nil {
		log.Print(err)
	}
}

func (s *Handler) clearAllTmp(exclude string) (err error) {
	realExcluded, err := filepath.Abs(exclude)
	if err != nil {
		return err
	}
	old, err := filepath.Glob(filepath.Join(os.TempDir(), "bla*"))
	if err != nil {
		return err
	}

	for _, path := range old {
		realPath, err := filepath.Abs(path)
		if err != nil {
			return err
		}
		if realPath == realExcluded {
			continue
		}
		log.Println("removing old public", realPath)
		os.RemoveAll(realPath)
	}
	return nil
}

func (s *Handler) saveAll() (err error) {

	s.publicPath, err = ioutil.TempDir("", "bla")
	if err != nil {
		return err
	}

	log.Print("Saving all docs...")
	s.mu.Lock()
	defer s.mu.Unlock()

	log.Println("Deleting public...")
	err = os.RemoveAll(s.publicPath)
	if err != nil {
		return
	}

	var f *os.File
	err = os.MkdirAll(s.publicPath, 0755)
	if err != nil {
		return
	}
	log.Printf("saving all docs...")
	for _, doc := range s.docs {
		f, err = os.Create(filepath.Join(s.publicPath, doc.SlugTitle))
		if err != nil {
			return
		}
		if err = s.tpl.ExecuteTemplate(f, "single",
			&singleData{s, doc.Title, doc}); err != nil {
			return
		}
		f.Close()
	}

	var docs []*Doc
	if len(s.sortDocs) > s.Cfg.HomeDocCount {
		docs = s.sortDocs[:s.Cfg.HomeDocCount]
	} else {
		docs = s.sortDocs
	}

	f, err = os.Create(filepath.Join(s.publicPath, "index.html"))
	if err != nil {
		return
	}
	defer f.Close()

	if err = s.tpl.ExecuteTemplate(f, "index",
		&mulDocData{s, "", docs}); err != nil {
		return
	}

	log.Printf("saving all tags...")
	err = os.MkdirAll(filepath.Join(s.publicPath, "tags"), 0755)
	if err != nil {
		return
	}

	for tagName, docs := range s.tags {
		f, err = os.Create(filepath.Join(s.publicPath, "/tags/", tagName))
		if err != nil {
			return
		}
		sort.Sort(docsByTime(docs))
		if err = s.tpl.ExecuteTemplate(f, "tag_page",
			&tagData{s, tagName, docs, tagName}); err != nil {
			return
		}
		f.Close()
	}

	log.Printf("linking all dir in %s", s.Cfg.RootPath)
	filepath.Walk(s.Cfg.RootPath, s.linkToPublic)
	log.Println("save completed")
	s.public = http.FileServer(http.Dir(s.publicPath))
	s.clearAllTmp(s.publicPath)
	return nil
}

func (s *Handler) linkToPublic(path string, info os.FileInfo, err error) error {
	if path == s.Cfg.RootPath {
		return nil
	}

	if strings.Count(path, "/")-strings.Count(s.Cfg.RootPath, "/") > 1 {
		return nil
	}

	switch base := filepath.Base(path); base {
	case "template", "docs", ".public", "":
		return nil
	default:
		realPath, err := filepath.Abs(path)
		if err != nil {
			return err
		}
		target := filepath.Join(s.publicPath, base)
		log.Printf("link %s -> %s", realPath, target)

		err = os.Symlink(realPath, target)
		if err != nil {
			if os.IsExist(err) {
				return nil
			}
			log.Fatal(err)
		}
	}
	return nil
}

func (s *Handler) loadData() {
	log.Print("Loading docs from:", s.docPath)

	s.mu.Lock()
	s.sortDocs = []*Doc{}
	s.docs = map[string]*Doc{}
	s.tags = map[string][]*Doc{}
	s.mu.Unlock()

	f, err := os.Open(s.docPath)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	err = filepath.Walk(s.docPath, s.docWalker)
	if err != nil {
		log.Print(err)
	}
	sort.Sort(docsByTime(s.sortDocs))
	log.Print("End Loading docs from:", s.docPath)
}

func (s *Handler) docWalker(p string, info os.FileInfo, err error) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	start := time.Now()
	if info.IsDir() || filepath.Ext(info.Name()) != ".md" {
		return nil
	}
	var f *os.File
	f, err = os.Open(p)
	if err != nil {
		return err
	}
	defer f.Close()

	var doc *Doc
	doc, err = newDoc(f)
	if err != nil {
		return err
	}
	if !doc.Public {
		log.Printf("doc:%s loaded but not public", p)
		return nil
	}

	doc.SlugTitle = path.Base(p)[0 : len(path.Base(p))-3]

	for _, t := range doc.Tags {
		s.tags[t] = append(s.tags[t], doc)
	}
	s.docs[doc.SlugTitle] = doc
	s.sortDocs = append(s.sortDocs, doc)
	log.Printf("loaded doc:%s in %s", doc.SlugTitle,
		time.Now().Sub(start).String())
	return nil
}

func (h *Handler) loadConfig() {

	f, err := os.Open(h.cfgPath)
	if err != nil && os.IsExist(err) {
		log.Panic(err)
	}
	defer f.Close()

	log.Print("loading config")
	cfg := DefaultConfig()
	dec := json.NewDecoder(f)
	err = dec.Decode(cfg)

	if err != nil {
		log.Panic(err)
	}

	h.templatePath = filepath.Join(cfg.RootPath, "template")
	h.docPath = filepath.Join(cfg.RootPath, "docs")

	h.Cfg = cfg
	log.Printf("%#v", *cfg)

}

func (s *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	if strings.HasPrefix(r.URL.Path, "/fs") {
		s.webfs.ServeHTTP(w, r)
	} else {
		s.public.ServeHTTP(w, r)
	}
}

func (s *Handler) loadTemplate() {
	s.mu.Lock()
	defer s.mu.Unlock()

	log.Printf("loding template:%s", s.templatePath)
	tpl, err := template.ParseGlob(s.templatePath + "/*.tmpl")
	if err != nil {
		log.Print(err)
		return
	}
	s.tpl = tpl
}
<commit_msg>update<commit_after>package bla

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"text/template"
	"time"

	"golang.org/x/net/webdav"

	"github.com/fsnotify/fsnotify"
)

// Initialize
type Handler struct {
	cfgPath string
	Cfg     *Config
	public  http.Handler
	webfs   http.Handler
	tpl     *template.Template

	publicPath   string
	templatePath string
	docPath      string

	mu       sync.RWMutex
	docs     map[string]*Doc
	sortDocs []*Doc
	tags     map[string][]*Doc
}

func NewHandler(cfgPath string) *Handler {

	h := &Handler{
		cfgPath: cfgPath,
		mu:      sync.RWMutex{},
	}

	h.loadConfig()
	h.watch()
	h.loadWebDav()

	return h
}

func (s *Handler) loadWebDav() {

	fs := webdav.Dir(s.Cfg.RootPath)
	ls := webdav.NewMemLS()

	handler := &webdav.Handler{
		Prefix:     "/fs",
		FileSystem: fs,
		LockSystem: ls,
	}
	s.webfs = handler
}

func (s *Handler) watch() {

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	go func() {
		// loadData minial interval is 1 second
		ticker := time.NewTicker(500 * time.Millisecond)
		mod := true
		rootChange := true

		for {
			select {
			case event := <-watcher.Events:
				switch ext := filepath.Ext(event.Name); ext {
				case ".md", ".json", ".tmpl":
					log.Println("modified file:", event.Name)
					mod = true
				case ".swp":
					continue
				}

				rootChange = true
			case err := <-watcher.Errors:
				log.Println("error:", err)
				return
			case <-ticker.C:
				if mod {
					mod = false
					s.loadData()
					s.loadTemplate()
				}

				if rootChange {
					rootChange = false
					err := s.saveAll()
					if err != nil {
						log.Fatal("can't save docs:", err)
					}
				}
			}
		}
	}()

	watcher.Add(s.Cfg.RootPath)
	watcher.Add(s.docPath)
	watcher.Add(s.templatePath)

	if err != nil {
		log.Print(err)
	}
}

func (s *Handler) clearAllTmp(exclude string) (err error) {
	realExcluded, err := filepath.Abs(exclude)
	if err != nil {
		return err
	}
	old, err := filepath.Glob(filepath.Join(os.TempDir(), "bla*"))
	if err != nil {
		return err
	}

	for _, path := range old {
		realPath, err := filepath.Abs(path)
		if err != nil {
			return err
		}
		if realPath == realExcluded {
			continue
		}
		log.Println("removing old public", realPath)
		os.RemoveAll(realPath)
	}
	return nil
}

func (s *Handler) saveAll() (err error) {

	s.publicPath, err = ioutil.TempDir("", "bla")
	if err != nil {
		return err
	}

	log.Print("Saving all docs...")
	s.mu.Lock()
	defer s.mu.Unlock()

	log.Println("Deleting public...")
	err = os.RemoveAll(s.publicPath)
	if err != nil {
		return
	}

	var f *os.File
	err = os.MkdirAll(s.publicPath, 0755)
	if err != nil {
		return
	}
	log.Printf("saving all docs...")
	for _, doc := range s.docs {
		f, err = os.Create(filepath.Join(s.publicPath, doc.SlugTitle))
		if err != nil {
			return
		}
		if err = s.tpl.ExecuteTemplate(f, "single",
			&singleData{s, doc.Title, doc}); err != nil {
			return
		}
		f.Close()
	}

	var docs []*Doc
	if len(s.sortDocs) > s.Cfg.HomeDocCount {
		docs = s.sortDocs[:s.Cfg.HomeDocCount]
	} else {
		docs = s.sortDocs
	}

	f, err = os.Create(filepath.Join(s.publicPath, "index.html"))
	if err != nil {
		return
	}
	defer f.Close()

	if err = s.tpl.ExecuteTemplate(f, "index",
		&mulDocData{s, "", docs}); err != nil {
		return
	}

	log.Printf("saving all tags...")
	err = os.MkdirAll(filepath.Join(s.publicPath, "tags"), 0755)
	if err != nil {
		return
	}

	for tagName, docs := range s.tags {
		f, err = os.Create(filepath.Join(s.publicPath, "/tags/", tagName))
		if err != nil {
			return
		}
		sort.Sort(docsByTime(docs))
		if err = s.tpl.ExecuteTemplate(f, "tag_page",
			&tagData{s, tagName, docs, tagName}); err != nil {
			return
		}
		f.Close()
	}

	log.Printf("linking all dir in %s", s.Cfg.RootPath)
	filepath.Walk(s.Cfg.RootPath, s.linkToPublic)
	log.Println("save completed")
	s.public = http.FileServer(http.Dir(s.publicPath))
	s.clearAllTmp(s.publicPath)
	return nil
}

func (s *Handler) linkToPublic(path string, info os.FileInfo, err error) error {
	if path == s.Cfg.RootPath {
		return nil
	}

	if strings.Count(path, "/")-strings.Count(s.Cfg.RootPath, "/") > 1 {
		return nil
	}

	switch base := filepath.Base(path); base {
	case "template", "docs", ".public", "":
		return nil
	default:
		realPath, err := filepath.Abs(path)
		if err != nil {
			return err
		}
		target := filepath.Join(s.publicPath, base)
		log.Printf("link %s -> %s", realPath, target)

		err = os.Symlink(realPath, target)
		if err != nil {
			if os.IsExist(err) {
				return nil
			}
			log.Fatal(err)
		}
	}
	return nil
}

func (s *Handler) loadData() {
	log.Print("Loading docs from:", s.docPath)

	s.mu.Lock()
	s.sortDocs = []*Doc{}
	s.docs = map[string]*Doc{}
	s.tags = map[string][]*Doc{}
	s.mu.Unlock()

	f, err := os.Open(s.docPath)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	err = filepath.Walk(s.docPath, s.docWalker)
	if err != nil {
		log.Print(err)
	}
	sort.Sort(docsByTime(s.sortDocs))
	log.Print("End Loading docs from:", s.docPath)
}

func (s *Handler) docWalker(p string, info os.FileInfo, err error) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	start := time.Now()
	if info.IsDir() || filepath.Ext(info.Name()) != ".md" {
		return nil
	}
	var f *os.File
	f, err = os.Open(p)
	if err != nil {
		return err
	}
	defer f.Close()

	var doc *Doc
	doc, err = newDoc(f)
	if err != nil {
		return err
	}
	if !doc.Public {
		log.Printf("doc:%s loaded but not public", p)
		return nil
	}

	doc.SlugTitle = path.Base(p)[0 : len(path.Base(p))-3]

	for _, t := range doc.Tags {
		s.tags[t] = append(s.tags[t], doc)
	}
	s.docs[doc.SlugTitle] = doc
	s.sortDocs = append(s.sortDocs, doc)
	log.Printf("loaded doc:%s in %s", doc.SlugTitle,
		time.Now().Sub(start).String())
	return nil
}

func (h *Handler) loadConfig() {

	f, err := os.Open(h.cfgPath)
	if err != nil && os.IsExist(err) {
		log.Panic(err)
	}
	defer f.Close()

	log.Print("loading config")
	cfg := DefaultConfig()
	dec := json.NewDecoder(f)
	err = dec.Decode(cfg)

	if err != nil {
		log.Panic(err)
	}

	h.templatePath = filepath.Join(cfg.RootPath, "template")
	h.docPath = filepath.Join(cfg.RootPath, "docs")

	h.Cfg = cfg
	log.Printf("%#v", *cfg)

}

func (s *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	if strings.HasPrefix(r.URL.Path, "/fs") {
		s.webfs.ServeHTTP(w, r)
	} else {
		s.public.ServeHTTP(w, r)
	}
}

func (s *Handler) loadTemplate() {
	s.mu.Lock()
	defer s.mu.Unlock()

	log.Printf("loding template:%s", s.templatePath)
	tpl, err := template.ParseGlob(s.templatePath + "/*.tmpl")
	if err != nil {
		log.Print(err)
		return
	}
	s.tpl = tpl
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"testing"

	"go.pedge.io/proto/server"
	"google.golang.org/grpc"

	"github.com/pachyderm/pachyderm"
	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/drive"
	"github.com/pachyderm/pachyderm/src/pfs/pfsutil"
	"github.com/pachyderm/pachyderm/src/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/pkg/require"
	"github.com/pachyderm/pachyderm/src/pkg/shard"
	"github.com/pachyderm/pachyderm/src/pkg/uuid"
)

const (
	shards = 32
)

var (
	port int32 = 30651
)

func TestSimple(t *testing.T) {
	t.Parallel()
	pfsClient, server := getClientAndServer(t)
	repo := uniqueString("TestSimple")
	require.NoError(t, pfsutil.CreateRepo(pfsClient, repo))
	commit1, err := pfsutil.StartCommit(pfsClient, repo, "", "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pfsClient, repo, commit1.ID, "foo", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	require.NoError(t, pfsutil.FinishCommit(pfsClient, repo, commit1.ID))
	commitInfos, err := pfsutil.ListCommit(pfsClient, []string{repo})
	require.NoError(t, err)
	require.Equal(t, 1, len(commitInfos))
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	commit2, err := pfsutil.StartCommit(pfsClient, repo, commit1.ID, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pfsClient, repo, commit2.ID, "foo", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	err = pfsutil.FinishCommit(pfsClient, repo, commit2.ID)
	require.NoError(t, err)
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit2.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\nfoo\n", buffer.String())

	// restart the server and make sure data is still there
	server.driver.Dump()
	restartServer(server, t)
	server.driver.Dump()
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit2.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\nfoo\n", buffer.String())
}

func TestBranch(t *testing.T) {
	t.Parallel()
	pfsClient, server := getClientAndServer(t)
	repo := uniqueString("TestBranch")
	require.NoError(t, pfsutil.CreateRepo(pfsClient, repo))
	commit1, err := pfsutil.StartCommit(pfsClient, repo, "", "master")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pfsClient, repo, "master", "foo", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	require.NoError(t, pfsutil.FinishCommit(pfsClient, repo, "master"))
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, "master", "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	branches, err := pfsutil.ListBranch(pfsClient, repo)
	require.NoError(t, err)
	require.Equal(t, commit1, branches[0].Commit)
	require.Equal(t, "master", branches[0].Branch)
	commit2, err := pfsutil.StartCommit(pfsClient, repo, "", "master")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pfsClient, repo, "master", "foo", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	err = pfsutil.FinishCommit(pfsClient, repo, "master")
	require.NoError(t, err)
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, "master", "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\nfoo\n", buffer.String())
	branches, err = pfsutil.ListBranch(pfsClient, repo)
	require.NoError(t, err)
	require.Equal(t, commit2, branches[0].Commit)
	require.Equal(t, "master", branches[0].Branch)

	// restart the server and make sure data is still there
	restartServer(server, t)
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, "master", "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\nfoo\n", buffer.String())
	branches, err = pfsutil.ListBranch(pfsClient, repo)
	require.NoError(t, err)
	require.Equal(t, commit2, branches[0].Commit)
	require.Equal(t, "master", branches[0].Branch)
}

func TestDisallowReadsDuringCommit(t *testing.T) {
	t.Parallel()
	pfsClient, server := getClientAndServer(t)
	repo := uniqueString("TestDisallowReadsDuringCommit")
	require.NoError(t, pfsutil.CreateRepo(pfsClient, repo))
	commit1, err := pfsutil.StartCommit(pfsClient, repo, "", "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pfsClient, repo, commit1.ID, "foo", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)

	// Make sure we can't get the file before the commit is finished
	var buffer bytes.Buffer
	require.YesError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "", buffer.String())

	require.NoError(t, pfsutil.FinishCommit(pfsClient, repo, commit1.ID))
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	commit2, err := pfsutil.StartCommit(pfsClient, repo, commit1.ID, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pfsClient, repo, commit2.ID, "foo", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	err = pfsutil.FinishCommit(pfsClient, repo, commit2.ID)
	require.NoError(t, err)
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit2.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\nfoo\n", buffer.String())

	// restart the server and make sure data is still there
	restartServer(server, t)
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit2.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\nfoo\n", buffer.String())
}

func getClientAndServer(t *testing.T) (pfs.APIClient, *internalAPIServer) {
	localPort := atomic.AddInt32(&port, 1)
	address := fmt.Sprintf("localhost:%d", localPort)
	driver, err := drive.NewDriver(address)
	require.NoError(t, err)
	root := uniqueString("/tmp/pach_test/run")
	t.Logf("root %s", root)
	blockAPIServer, err := NewLocalBlockAPIServer(root)
	require.NoError(t, err)
	sharder := shard.NewLocalSharder(address, shards)
	hasher := pfs.NewHasher(shards, 1)
	dialer := grpcutil.NewDialer(grpc.WithInsecure())
	apiServer := NewAPIServer(hasher, shard.NewRouter(sharder, dialer, address))
	internalAPIServer := newInternalAPIServer(hasher, shard.NewRouter(sharder, dialer, address), driver)
	ready := make(chan bool)
	go func() {
		err := protoserver.Serve(
			func(s *grpc.Server) {
				pfs.RegisterAPIServer(s, apiServer)
				pfs.RegisterInternalAPIServer(s, internalAPIServer)
				pfs.RegisterBlockAPIServer(s, blockAPIServer)
				close(ready)
			},
			protoserver.ServeOptions{Version: pachyderm.Version},
			protoserver.ServeEnv{GRPCPort: uint16(localPort)},
		)
		require.NoError(t, err)
	}()
	<-ready
	for i := 0; i < shards; i++ {
		require.NoError(t, internalAPIServer.AddShard(uint64(i)))
	}
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	require.NoError(t, err)
	return pfs.NewAPIClient(clientConn), internalAPIServer
}

func restartServer(server *internalAPIServer, t *testing.T) {
	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < shards; i++ {
		i := i
		go func() {
			wg.Add(1)
			defer wg.Done()
			require.NoError(t, server.DeleteShard(uint64(i)))
			require.NoError(t, server.AddShard(uint64(i)))
		}()
	}
}

func uniqueString(prefix string) string {
	return prefix + "." + uuid.NewWithoutDashes()[0:12]
}
<commit_msg>server_test now directly tests block server<commit_after>package server

import (
	"bytes"
	"fmt"
	"io"
	"strings"
	"sync"
	"sync/atomic"
	"testing"

	"golang.org/x/net/context"

	"go.pedge.io/proto/server"
	"google.golang.org/grpc"

	"github.com/pachyderm/pachyderm"
	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/drive"
	"github.com/pachyderm/pachyderm/src/pfs/pfsutil"
	"github.com/pachyderm/pachyderm/src/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/pkg/require"
	"github.com/pachyderm/pachyderm/src/pkg/shard"
	"github.com/pachyderm/pachyderm/src/pkg/uuid"
)

const (
	shards = 1
)

var (
	port int32 = 30651
)

func TestBlock(t *testing.T) {
	t.Parallel()
	blockClient := getBlockClient(t)
	_, err := blockClient.CreateDiff(
		context.Background(),
		&pfs.DiffInfo{
			Diff: pfsutil.NewDiff("foo", "", 0),
		})
	require.NoError(t, err)
	_, err = blockClient.CreateDiff(
		context.Background(),
		&pfs.DiffInfo{
			Diff: pfsutil.NewDiff("foo", "c1", 0),
		})
	require.NoError(t, err)
	_, err = blockClient.CreateDiff(
		context.Background(),
		&pfs.DiffInfo{
			Diff: pfsutil.NewDiff("foo", "c2", 0),
		})
	require.NoError(t, err)
	listDiffClient, err := blockClient.ListDiff(
		context.Background(),
		&pfs.ListDiffRequest{Shard: 0},
	)
	require.NoError(t, err)
	var diffInfos []*pfs.DiffInfo
	for {
		diffInfo, err := listDiffClient.Recv()
		if err == io.EOF {
			break
		} else {
			require.NoError(t, err)
		}
		diffInfos = append(diffInfos, diffInfo)
	}
	require.Equal(t, 3, len(diffInfos))
}

func TestSimple(t *testing.T) {
	t.Parallel()
	pfsClient, server := getClientAndServer(t)
	repo := uniqueString("TestSimple")
	require.NoError(t, pfsutil.CreateRepo(pfsClient, repo))
	commit1, err := pfsutil.StartCommit(pfsClient, repo, "", "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pfsClient, repo, commit1.ID, "foo", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	require.NoError(t, pfsutil.FinishCommit(pfsClient, repo, commit1.ID))
	commitInfos, err := pfsutil.ListCommit(pfsClient, []string{repo})
	require.NoError(t, err)
	require.Equal(t, 1, len(commitInfos))
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	commit2, err := pfsutil.StartCommit(pfsClient, repo, commit1.ID, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pfsClient, repo, commit2.ID, "foo", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	err = pfsutil.FinishCommit(pfsClient, repo, commit2.ID)
	require.NoError(t, err)
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit2.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\nfoo\n", buffer.String())

	// restart the server and make sure data is still there
	server.driver.Dump()
	restartServer(server, t)
	server.driver.Dump()
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit2.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\nfoo\n", buffer.String())
}

func TestBranch(t *testing.T) {
	t.Parallel()
	pfsClient, server := getClientAndServer(t)
	repo := uniqueString("TestBranch")
	require.NoError(t, pfsutil.CreateRepo(pfsClient, repo))
	commit1, err := pfsutil.StartCommit(pfsClient, repo, "", "master")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pfsClient, repo, "master", "foo", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	require.NoError(t, pfsutil.FinishCommit(pfsClient, repo, "master"))
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, "master", "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	branches, err := pfsutil.ListBranch(pfsClient, repo)
	require.NoError(t, err)
	require.Equal(t, commit1, branches[0].Commit)
	require.Equal(t, "master", branches[0].Branch)
	commit2, err := pfsutil.StartCommit(pfsClient, repo, "", "master")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pfsClient, repo, "master", "foo", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	err = pfsutil.FinishCommit(pfsClient, repo, "master")
	require.NoError(t, err)
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, "master", "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\nfoo\n", buffer.String())
	branches, err = pfsutil.ListBranch(pfsClient, repo)
	require.NoError(t, err)
	require.Equal(t, commit2, branches[0].Commit)
	require.Equal(t, "master", branches[0].Branch)

	// restart the server and make sure data is still there
	restartServer(server, t)
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, "master", "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\nfoo\n", buffer.String())
	branches, err = pfsutil.ListBranch(pfsClient, repo)
	require.NoError(t, err)
	require.Equal(t, commit2, branches[0].Commit)
	require.Equal(t, "master", branches[0].Branch)
}

func TestDisallowReadsDuringCommit(t *testing.T) {
	t.Parallel()
	pfsClient, server := getClientAndServer(t)
	repo := uniqueString("TestDisallowReadsDuringCommit")
	require.NoError(t, pfsutil.CreateRepo(pfsClient, repo))
	commit1, err := pfsutil.StartCommit(pfsClient, repo, "", "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pfsClient, repo, commit1.ID, "foo", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)

	// Make sure we can't get the file before the commit is finished
	var buffer bytes.Buffer
	require.YesError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "", buffer.String())

	require.NoError(t, pfsutil.FinishCommit(pfsClient, repo, commit1.ID))
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	commit2, err := pfsutil.StartCommit(pfsClient, repo, commit1.ID, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pfsClient, repo, commit2.ID, "foo", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	err = pfsutil.FinishCommit(pfsClient, repo, commit2.ID)
	require.NoError(t, err)
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit2.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\nfoo\n", buffer.String())

	// restart the server and make sure data is still there
	restartServer(server, t)
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit1.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pfsClient, repo, commit2.ID, "foo", 0, 0, "", nil, &buffer))
	require.Equal(t, "foo\nfoo\n", buffer.String())
}

func getBlockClient(t *testing.T) pfs.BlockAPIClient {
	localPort := atomic.AddInt32(&port, 1)
	address := fmt.Sprintf("localhost:%d", localPort)
	root := uniqueString("/tmp/pach_test/run")
	t.Logf("root %s", root)
	blockAPIServer, err := NewLocalBlockAPIServer(root)
	require.NoError(t, err)
	ready := make(chan bool)
	go func() {
		err := protoserver.Serve(
			func(s *grpc.Server) {
				pfs.RegisterBlockAPIServer(s, blockAPIServer)
				close(ready)
			},
			protoserver.ServeOptions{Version: pachyderm.Version},
			protoserver.ServeEnv{GRPCPort: uint16(localPort)},
		)
		require.NoError(t, err)
	}()
	<-ready
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	return pfs.NewBlockAPIClient(clientConn)
}

func getClientAndServer(t *testing.T) (pfs.APIClient, *internalAPIServer) {
	localPort := atomic.AddInt32(&port, 1)
	address := fmt.Sprintf("localhost:%d", localPort)
	driver, err := drive.NewDriver(address)
	require.NoError(t, err)
	root := uniqueString("/tmp/pach_test/run")
	t.Logf("root %s", root)
	blockAPIServer, err := NewLocalBlockAPIServer(root)
	require.NoError(t, err)
	sharder := shard.NewLocalSharder(address, shards)
	hasher := pfs.NewHasher(shards, 1)
	dialer := grpcutil.NewDialer(grpc.WithInsecure())
	apiServer := NewAPIServer(hasher, shard.NewRouter(sharder, dialer, address))
	internalAPIServer := newInternalAPIServer(hasher, shard.NewRouter(sharder, dialer, address), driver)
	ready := make(chan bool)
	go func() {
		err := protoserver.Serve(
			func(s *grpc.Server) {
				pfs.RegisterAPIServer(s, apiServer)
				pfs.RegisterInternalAPIServer(s, internalAPIServer)
				pfs.RegisterBlockAPIServer(s, blockAPIServer)
				close(ready)
			},
			protoserver.ServeOptions{Version: pachyderm.Version},
			protoserver.ServeEnv{GRPCPort: uint16(localPort)},
		)
		require.NoError(t, err)
	}()
	<-ready
	for i := 0; i < shards; i++ {
		require.NoError(t, internalAPIServer.AddShard(uint64(i)))
	}
	clientConn, err := grpc.Dial(address, grpc.WithInsecure())
	require.NoError(t, err)
	return pfs.NewAPIClient(clientConn), internalAPIServer
}

func restartServer(server *internalAPIServer, t *testing.T) {
	var wg sync.WaitGroup
	defer wg.Wait()
	for i := 0; i < shards; i++ {
		i := i
		go func() {
			wg.Add(1)
			defer wg.Done()
			require.NoError(t, server.DeleteShard(uint64(i)))
			require.NoError(t, server.AddShard(uint64(i)))
		}()
	}
}

func uniqueString(prefix string) string {
	return prefix + "." + uuid.NewWithoutDashes()[0:12]
}
<|endoftext|>