<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package revision

import (
	"context"
	"fmt"
	"sync"
	"time"

	"github.com/google/go-containerregistry/pkg/authn/k8schain"
	"go.uber.org/zap"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/util/workqueue"
	v1 "knative.dev/serving/pkg/apis/serving/v1"
)

// imageResolver is an interface used mostly to mock digestResolver for tests.
type imageResolver interface {
	Resolve(ctx context.Context, image string, opt k8schain.Options, registriesToSkip sets.String) (string, error)
}

// backgroundResolver performs background downloads of image digests.
type backgroundResolver struct {
	logger *zap.SugaredLogger

	resolver imageResolver
	enqueue  func(types.NamespacedName)

	queue workqueue.Interface

	mu      sync.Mutex
	results map[types.NamespacedName]*resolveResult
}

// resolveResult is the overall result for a particular revision. We create a
// workItem for each container we need to resolve for the overall result.
type resolveResult struct {
	// these fields are immutable afer creation, so can be accessed without a lock.
	opt                k8schain.Options
	registriesToSkip   sets.String
	completionCallback func()

	// these fields can be written concurrently, so should only be accessed while
	// holding the backgroundResolver mutex.
	statuses  []v1.ContainerStatus
	err       error
	remaining int
}

// workItem is a single task submitted to the queue, to resolve a single image
// for a resolveResult.
type workItem struct {
	result  *resolveResult
	timeout time.Duration

	name  string
	image string
	index int
}

func newBackgroundResolver(logger *zap.SugaredLogger, resolver imageResolver, enqueue func(types.NamespacedName)) *backgroundResolver {
	r := &backgroundResolver{
		logger: logger,

		resolver: resolver,
		enqueue:  enqueue,

		results: make(map[types.NamespacedName]*resolveResult),
		queue:   workqueue.NewNamed("digests"),
	}

	return r
}

// Start starts the worker threads and runs maxInFlight workers until the stop
// channel is closed. It returns a done channel which will be closed when all
// workers have exited.
func (r *backgroundResolver) Start(stop <-chan struct{}, maxInFlight int) (done chan struct{}) {
	var wg sync.WaitGroup

	// Run the worker threads.
	wg.Add(maxInFlight)
	for i := 0; i < maxInFlight; i++ {
		go func() {
			defer wg.Done()
			for {
				item, shutdown := r.queue.Get()
				if shutdown {
					return
				}

				rrItem, ok := item.(*workItem)
				if !ok {
					r.logger.Fatalf("Unexpected work item type: want: %T, got: %T", &workItem{}, item)
				}

				r.processWorkItem(rrItem)
			}
		}()
	}

	// Shut down the queue once the stop channel is closed, this will cause all
	// the workers to exit.
	go func() {
		<-stop
		r.queue.ShutDown()
	}()

	// Return a done channel which is closed once all workers exit.
	done = make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	return done
}

// Resolve is intended to be called from a reconciler to resolve a revision. If
// the resolver already has the digest in cache it is returned immediately, if
// it does not and no resolution is already in flight a resolution is triggered
// in the background.
// If this method returns `nil, nil` this implies a resolve was triggered or is
// already in progress, so the reconciler should exit and wait for the revision
// to be re-enqueued when the result is ready.
func (r *backgroundResolver) Resolve(rev *v1.Revision, opt k8schain.Options, registriesToSkip sets.String, timeout time.Duration) ([]v1.ContainerStatus, error) {
	r.mu.Lock()
	defer r.mu.Unlock()

	name := types.NamespacedName{
		Name:      rev.Name,
		Namespace: rev.Namespace,
	}

	result, inFlight := r.results[name]
	if !inFlight {
		r.addWorkItems(rev, name, opt, registriesToSkip, timeout)
		return nil, nil
	}

	if !result.ready() {
		return nil, nil
	}

	ret := r.results[name]
	return ret.statuses, ret.err
}

// addWorkItems adds a digest resolve item to the queue for each container in the revision.
// This is expected to be called with the mutex locked.
func (r *backgroundResolver) addWorkItems(rev *v1.Revision, name types.NamespacedName, opt k8schain.Options, registriesToSkip sets.String, timeout time.Duration) {
	r.results[name] = &resolveResult{
		opt:              opt,
		registriesToSkip: registriesToSkip,
		statuses:         make([]v1.ContainerStatus, len(rev.Spec.Containers)),
		remaining:        len(rev.Spec.Containers),
		completionCallback: func() {
			r.enqueue(name)
		},
	}

	for i := range rev.Spec.Containers {
		image := rev.Spec.Containers[i].Image

		r.queue.Add(&workItem{
			result:  r.results[name],
			timeout: timeout,
			name:    rev.Spec.Containers[i].Name,
			image:   image,
			index:   i,
		})
	}
}

// processWorkItem runs a single image digest resolution and stores the result
// in the resolveResult. If this completes the work for the revision, the
// completionCallback is called.
func (r *backgroundResolver) processWorkItem(item *workItem) {
	defer r.queue.Done(item)

	ctx, cancel := context.WithTimeout(context.Background(), item.timeout)
	defer cancel()

	resolvedDigest, resolveErr := r.resolver.Resolve(ctx, item.image, item.result.opt, item.result.registriesToSkip)

	// lock after the resolve because we don't want to block parallel resolves,
	// just storing the result.
	r.mu.Lock()
	defer r.mu.Unlock()

	// If we're already ready we don't want to callback twice.
	// This can happen if an image resolve completes but we've already reported
	// an error from another image in the result.
	if item.result.ready() {
		return
	}

	if resolveErr != nil {
		item.result.statuses = nil
		item.result.err = containerMissingError{image: item.image, cause: resolveErr}
		item.result.completionCallback()
		return
	}

	item.result.remaining--
	item.result.statuses[item.index] = v1.ContainerStatus{
		Name:        item.name,
		ImageDigest: resolvedDigest,
	}

	if item.result.ready() {
		item.result.completionCallback()
	}
}

// Clear removes any cached results for the revision. This should be called
// when the revision is deleted or once the revision's ContainerStatus has been
// set.
func (r *backgroundResolver) Clear(name types.NamespacedName) {
	r.mu.Lock()
	defer r.mu.Unlock()

	// delete the map entry, the result can still be accessed via the pointer in
	// the workItem so processWorkItem doesn't care about this, but once the
	// workItem is done the result will be GC-ed.
	delete(r.results, name)
}

func (r *resolveResult) ready() bool {
	return r.remaining == 0 || r.err != nil
}

// containerMissingError converts an error in to the expected format for the
// RevisionContainerMissing condition.
type containerMissingError struct {
	image string
	cause error
}

func (e containerMissingError) Error() string {
	return v1.RevisionContainerMissingMessage(e.image, fmt.Sprint("failed to resolve image to digest:", e.cause))
}

func (e containerMissingError) Unwrap() error {
	return e.cause
}
<commit_msg>Drop custom error type for error wrapping (#9635)<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package revision

import (
	"context"
	"fmt"
	"sync"
	"time"

	"github.com/google/go-containerregistry/pkg/authn/k8schain"
	"go.uber.org/zap"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/util/workqueue"
	v1 "knative.dev/serving/pkg/apis/serving/v1"
)

// imageResolver is an interface used mostly to mock digestResolver for tests.
type imageResolver interface {
	Resolve(ctx context.Context, image string, opt k8schain.Options, registriesToSkip sets.String) (string, error)
}

// backgroundResolver performs background downloads of image digests.
type backgroundResolver struct {
	logger *zap.SugaredLogger

	resolver imageResolver
	enqueue  func(types.NamespacedName)

	queue workqueue.Interface

	mu      sync.Mutex
	results map[types.NamespacedName]*resolveResult
}

// resolveResult is the overall result for a particular revision. We create a
// workItem for each container we need to resolve for the overall result.
type resolveResult struct {
	// these fields are immutable afer creation, so can be accessed without a lock.
	opt                k8schain.Options
	registriesToSkip   sets.String
	completionCallback func()

	// these fields can be written concurrently, so should only be accessed while
	// holding the backgroundResolver mutex.
	statuses  []v1.ContainerStatus
	err       error
	remaining int
}

// workItem is a single task submitted to the queue, to resolve a single image
// for a resolveResult.
type workItem struct {
	result  *resolveResult
	timeout time.Duration

	name  string
	image string
	index int
}

func newBackgroundResolver(logger *zap.SugaredLogger, resolver imageResolver, enqueue func(types.NamespacedName)) *backgroundResolver {
	r := &backgroundResolver{
		logger: logger,

		resolver: resolver,
		enqueue:  enqueue,

		results: make(map[types.NamespacedName]*resolveResult),
		queue:   workqueue.NewNamed("digests"),
	}

	return r
}

// Start starts the worker threads and runs maxInFlight workers until the stop
// channel is closed. It returns a done channel which will be closed when all
// workers have exited.
func (r *backgroundResolver) Start(stop <-chan struct{}, maxInFlight int) (done chan struct{}) {
	var wg sync.WaitGroup

	// Run the worker threads.
	wg.Add(maxInFlight)
	for i := 0; i < maxInFlight; i++ {
		go func() {
			defer wg.Done()
			for {
				item, shutdown := r.queue.Get()
				if shutdown {
					return
				}

				rrItem, ok := item.(*workItem)
				if !ok {
					r.logger.Fatalf("Unexpected work item type: want: %T, got: %T", &workItem{}, item)
				}

				r.processWorkItem(rrItem)
			}
		}()
	}

	// Shut down the queue once the stop channel is closed, this will cause all
	// the workers to exit.
	go func() {
		<-stop
		r.queue.ShutDown()
	}()

	// Return a done channel which is closed once all workers exit.
	done = make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	return done
}

// Resolve is intended to be called from a reconciler to resolve a revision. If
// the resolver already has the digest in cache it is returned immediately, if
// it does not and no resolution is already in flight a resolution is triggered
// in the background.
// If this method returns `nil, nil` this implies a resolve was triggered or is
// already in progress, so the reconciler should exit and wait for the revision
// to be re-enqueued when the result is ready.
func (r *backgroundResolver) Resolve(rev *v1.Revision, opt k8schain.Options, registriesToSkip sets.String, timeout time.Duration) ([]v1.ContainerStatus, error) {
	r.mu.Lock()
	defer r.mu.Unlock()

	name := types.NamespacedName{
		Name:      rev.Name,
		Namespace: rev.Namespace,
	}

	result, inFlight := r.results[name]
	if !inFlight {
		r.addWorkItems(rev, name, opt, registriesToSkip, timeout)
		return nil, nil
	}

	if !result.ready() {
		return nil, nil
	}

	ret := r.results[name]
	return ret.statuses, ret.err
}

// addWorkItems adds a digest resolve item to the queue for each container in the revision.
// This is expected to be called with the mutex locked.
func (r *backgroundResolver) addWorkItems(rev *v1.Revision, name types.NamespacedName, opt k8schain.Options, registriesToSkip sets.String, timeout time.Duration) {
	r.results[name] = &resolveResult{
		opt:              opt,
		registriesToSkip: registriesToSkip,
		statuses:         make([]v1.ContainerStatus, len(rev.Spec.Containers)),
		remaining:        len(rev.Spec.Containers),
		completionCallback: func() {
			r.enqueue(name)
		},
	}

	for i := range rev.Spec.Containers {
		image := rev.Spec.Containers[i].Image

		r.queue.Add(&workItem{
			result:  r.results[name],
			timeout: timeout,
			name:    rev.Spec.Containers[i].Name,
			image:   image,
			index:   i,
		})
	}
}

// processWorkItem runs a single image digest resolution and stores the result
// in the resolveResult. If this completes the work for the revision, the
// completionCallback is called.
func (r *backgroundResolver) processWorkItem(item *workItem) {
	defer r.queue.Done(item)

	ctx, cancel := context.WithTimeout(context.Background(), item.timeout)
	defer cancel()

	resolvedDigest, resolveErr := r.resolver.Resolve(ctx, item.image, item.result.opt, item.result.registriesToSkip)

	// lock after the resolve because we don't want to block parallel resolves,
	// just storing the result.
	r.mu.Lock()
	defer r.mu.Unlock()

	// If we're already ready we don't want to callback twice.
	// This can happen if an image resolve completes but we've already reported
	// an error from another image in the result.
	if item.result.ready() {
		return
	}

	if resolveErr != nil {
		item.result.statuses = nil
		item.result.err = fmt.Errorf("%s: %w", v1.RevisionContainerMissingMessage(item.image, "failed to resolve image to digest"), resolveErr)
		item.result.completionCallback()
		return
	}

	item.result.remaining--
	item.result.statuses[item.index] = v1.ContainerStatus{
		Name:        item.name,
		ImageDigest: resolvedDigest,
	}

	if item.result.ready() {
		item.result.completionCallback()
	}
}

// Clear removes any cached results for the revision. This should be called
// when the revision is deleted or once the revision's ContainerStatus has been
// set.
func (r *backgroundResolver) Clear(name types.NamespacedName) {
	r.mu.Lock()
	defer r.mu.Unlock()

	// delete the map entry, the result can still be accessed via the pointer in
	// the workItem so processWorkItem doesn't care about this, but once the
	// workItem is done the result will be GC-ed.
	delete(r.results, name)
}

func (r *resolveResult) ready() bool {
	return r.remaining == 0 || r.err != nil
}
<|endoftext|>


<commit_before>//go:build go1.16
// +build go1.16

package embed

import (
	"errors"
	"io/fs"
	"path/filepath"
	"strings"

	"github.com/webx-top/echo"
)

func NewFileSystems() FileSystems {
	return FileSystems{}
}

type FileSystems []fs.FS

func (f FileSystems) Open(name string) (file fs.File, err error) {
	for _, fileSystem := range f {
		file, err = fileSystem.Open(name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) ReadFile(name string) (content []byte, err error) {
	for _, fileSystem := range f {
		content, err = fs.ReadFile(fileSystem, name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) ReadDir(name string) (dirs []fs.DirEntry, err error) {
	for _, fileSystem := range f {
		dirs, err = fs.ReadDir(fileSystem, name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) Sub(name string) (sub fs.FS, err error) {
	for _, fileSystem := range f {
		sub, err = fs.Sub(fileSystem, name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) WalkDir(name string, fn fs.WalkDirFunc) (err error) {
	for _, fileSystem := range f {
		err = fs.WalkDir(fileSystem, name, fn)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) Stat(name string) (fi fs.FileInfo, err error) {
	for _, fileSystem := range f {
		fi, err = fs.Stat(fileSystem, name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) Glob(pattern string) (matches []string, err error) {
	for _, fileSystem := range f {
		matches, err = fs.Glob(fileSystem, pattern)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) Size() int {
	return len(f)
}

func (f FileSystems) IsEmpty() bool {
	return f.Size() == 0
}

func (f *FileSystems) Register(fileSystem fs.FS) {
	*f = append(*f, fileSystem)
}

// File
// e.Get(`/*`, File(customFS))
func File(fs FileSystems, configs ...Config) func(c echo.Context) error {
	config := DefaultConfig
	if len(configs) > 0 {
		config = configs[0]
		if len(config.Index) == 0 {
			config.Index = DefaultConfig.Index
		}
		if len(config.Prefix) > 0 {
			config.Prefix = strings.TrimPrefix(config.Prefix, `/`)
		}
		if len(config.Prefix) > 0 {
			if !strings.HasSuffix(config.Prefix, `/`) {
				config.Prefix += `/`
			}
		}
		if config.FilePath == nil {
			config.FilePath = DefaultConfig.FilePath
		}
	}
	return func(c echo.Context) error {
		file, err := config.FilePath(c)
		if err != nil {
			return err
		}
		if len(file) == 0 {
			file = config.Index
		}
		if len(config.Prefix) > 0 {
			file = config.Prefix + file
		}
		f, err := fs.Open(file)
		if err != nil {
			return echo.ErrNotFound
		}
		defer func() {
			if f != nil {
				f.Close()
			}
		}()
		fi, err := f.Stat()
		if err != nil {
			return err
		}
		if fi.IsDir() {
			f.Close()

			file = filepath.Join(file, config.Index)
			if f, err = fs.Open(file); err != nil {
				return echo.ErrNotFound
			}

			if fi, err = f.Stat(); err != nil {
				return err
			}
		}
		return c.ServeContent(f, fi.Name(), fi.ModTime())
	}
}
<commit_msg>update<commit_after>//go:build go1.16
// +build go1.16

package embed

import (
	"errors"
	"io/fs"
	"path/filepath"
	"strings"

	"github.com/webx-top/echo"
)

func NewFileSystems() FileSystems {
	return FileSystems{}
}

type FileSystems []fs.FS

func (f FileSystems) Open(name string) (file fs.File, err error) {
	for _, fileSystem := range f {
		file, err = fileSystem.Open(name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) ReadFile(name string) (content []byte, err error) {
	for _, fileSystem := range f {
		content, err = fs.ReadFile(fileSystem, name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) ReadDir(name string) (dirs []fs.DirEntry, err error) {
	unique := map[string]struct{}{}
	for _, fileSystem := range f {
		var _dirs []fs.DirEntry
		_dirs, err = fs.ReadDir(fileSystem, name)
		if err != nil && !errors.Is(err, fs.ErrNotExist) {
			return
		}
		err = nil
		for _, dir := range _dirs {
			if _, ok := unique[dir.Name()]; ok {
				continue
			}
			unique[dir.Name()] = struct{}{}
			dirs = append(dirs, dir)
		}
	}
	return
}

func (f FileSystems) Sub(name string) (sub fs.FS, err error) {
	for _, fileSystem := range f {
		sub, err = fs.Sub(fileSystem, name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) WalkDir(name string, fn fs.WalkDirFunc) (err error) {
	for _, fileSystem := range f {
		err = fs.WalkDir(fileSystem, name, fn)
		if err != nil && !errors.Is(err, fs.ErrNotExist) {
			return
		}
		err = nil
	}
	return
}

func (f FileSystems) Stat(name string) (fi fs.FileInfo, err error) {
	for _, fileSystem := range f {
		fi, err = fs.Stat(fileSystem, name)
		if err == nil || !errors.Is(err, fs.ErrNotExist) {
			return
		}
	}
	return
}

func (f FileSystems) Glob(pattern string) (matches []string, err error) {
	unique := map[string]struct{}{}
	for _, fileSystem := range f {
		var mch []string
		mch, err = fs.Glob(fileSystem, pattern)
		if err != nil && !errors.Is(err, fs.ErrNotExist) {
			return
		}
		err = nil
		for _, match := range mch {
			if _, ok := unique[match]; ok {
				continue
			}
			unique[match] = struct{}{}
			matches = append(matches, match)
		}
	}
	return
}

func (f FileSystems) Size() int {
	return len(f)
}

func (f FileSystems) IsEmpty() bool {
	return f.Size() == 0
}

func (f *FileSystems) Register(fileSystem fs.FS) {
	*f = append(*f, fileSystem)
}

// File
// e.Get(`/*`, File(customFS))
func File(fs FileSystems, configs ...Config) func(c echo.Context) error {
	config := DefaultConfig
	if len(configs) > 0 {
		config = configs[0]
		if len(config.Index) == 0 {
			config.Index = DefaultConfig.Index
		}
		if len(config.Prefix) > 0 {
			config.Prefix = strings.TrimPrefix(config.Prefix, `/`)
		}
		if len(config.Prefix) > 0 {
			if !strings.HasSuffix(config.Prefix, `/`) {
				config.Prefix += `/`
			}
		}
		if config.FilePath == nil {
			config.FilePath = DefaultConfig.FilePath
		}
	}
	return func(c echo.Context) error {
		file, err := config.FilePath(c)
		if err != nil {
			return err
		}
		if len(file) == 0 {
			file = config.Index
		}
		if len(config.Prefix) > 0 {
			file = config.Prefix + file
		}
		f, err := fs.Open(file)
		if err != nil {
			return echo.ErrNotFound
		}
		defer func() {
			if f != nil {
				f.Close()
			}
		}()
		fi, err := f.Stat()
		if err != nil {
			return err
		}
		if fi.IsDir() {
			f.Close()

			file = filepath.Join(file, config.Index)
			if f, err = fs.Open(file); err != nil {
				return echo.ErrNotFound
			}

			if fi, err = f.Stat(); err != nil {
				return err
			}
		}
		return c.ServeContent(f, fi.Name(), fi.ModTime())
	}
}
<|endoftext|>


<commit_before>package test

import (
	"log"
	"time"

	r "github.com/dancannon/gorethink"
	"github.com/javinc/mango/module"
)

const tableName = "test"

// Resource test
type Resource struct {
}

// Object test
type Object struct {
	ID          string    `gorethink:"id,omitempty" json:"id,omitempty"`
	Title       string    `gorethink:"title,omitempty" json:"title,omitempty"`
	Description string    `gorethink:"description,omitempty" json:"description,omitempty"`
	CreatedAt   time.Time `gorethink:"created_at,omitempty" json:"created_at,omitempty"`
	UpdatedAt   time.Time `gorethink:"updated_at,omitempty" json:"updated_at,omitempty"`
	Deleted     bool      `gorethink:"deleted,omitempty" json:"deleted"`
}

// Find test
func (t *Resource) Find() []Object {
	data := []Object{}

	res, err := r.Table(tableName).
		OrderBy(r.Desc("created_at")).
		Run(module.RSession)
	if err != nil {
		log.Fatalln(err.Error())
	}

	err = res.All(&data)
	if err != nil {
		log.Fatalln(err.Error())
	}

	return data
}

// Get test
func (t *Resource) Get(id string) Object {
	data := Object{}

	res, err := r.Table(tableName).
		Get(id).Run(module.RSession)
	if err != nil {
		log.Fatalln(err.Error())
	}

	err = res.One(&data)
	if err != nil {
		log.Fatalln(err.Error())
	}

	return data
}

// Create test
func (t *Resource) Create(p Object) Object {
	// meta data
	p.CreatedAt = time.Now()
	p.UpdatedAt = time.Now()
	p.Deleted = false

	// insert to database
	_, err := r.Table(tableName).Insert(p).Run(module.RSession)
	if err != nil {
		log.Fatalln(err.Error())

		return Object{}
	}

	return p
}

// Update test
func (t *Resource) Update(p Object, id string) Object {
	// check item if exists
	payload := t.Get(id)
	if payload.ID == "" {
		log.Fatalln("not exists")

		return Object{}
	}

	// meta data
	p.UpdatedAt = time.Now()

	// insert to database
	_, err := r.Table(tableName).Get(id).Update(p).Run(module.RSession)
	if err != nil {
		log.Fatalln(err.Error())

		return Object{}
	}

	return p
}

// Remove test
func (t *Resource) Remove(id string) bool {
	// check item if exists
	payload := t.Get(id)
	if payload.ID == "" {
		log.Fatalln("not exists")

		return false
	}

	// meta data
	p := Object{}
	p.UpdatedAt = time.Now()
	p.Deleted = true

	// insert to database
	_, err := r.Table(tableName).Get(id).Update(p).Run(module.RSession)
	if err != nil {
		log.Fatalln(err.Error())

		return false
	}

	return true
}

// HardRemove test
func (t *Resource) HardRemove(id string) bool {
	// check item if exists
	payload := t.Get(id)
	if payload.ID == "" {
		log.Fatalln("not exists")

		return false
	}

	// insert to database
	_, err := r.Table(tableName).Get(id).Delete().Run(module.RSession)
	if err != nil {
		log.Fatalln(err.Error())

		return false
	}

	return true
}
<commit_msg>not soft deleting<commit_after>package test

import (
	"log"
	"time"

	r "github.com/dancannon/gorethink"
	"github.com/javinc/mango/module"
)

const tableName = "test"

// Resource test
type Resource struct {
}

// Object test
type Object struct {
	ID          string    `gorethink:"id,omitempty" json:"id,omitempty"`
	Title       string    `gorethink:"title,omitempty" json:"title,omitempty"`
	Description string    `gorethink:"description,omitempty" json:"description,omitempty"`
	CreatedAt   time.Time `gorethink:"created_at,omitempty" json:"created_at,omitempty"`
	UpdatedAt   time.Time `gorethink:"updated_at,omitempty" json:"updated_at,omitempty"`
}

// Find test
func (t *Resource) Find() []Object {
	data := []Object{}

	res, err := r.Table(tableName).
		OrderBy(r.Desc("created_at")).
		Run(module.RSession)
	if err != nil {
		log.Fatalln(err.Error())
	}

	err = res.All(&data)
	if err != nil {
		log.Fatalln(err.Error())
	}

	return data
}

// Get test
func (t *Resource) Get(id string) Object {
	data := Object{}

	res, err := r.Table(tableName).
		Get(id).Run(module.RSession)
	if err != nil {
		log.Fatalln(err.Error())
	}

	err = res.One(&data)
	if err != nil {
		log.Fatalln(err.Error())
	}

	return data
}

// Create test
func (t *Resource) Create(p Object) Object {
	// meta data
	p.CreatedAt = time.Now()
	p.UpdatedAt = time.Now()

	// insert to database
	_, err := r.Table(tableName).Insert(p).Run(module.RSession)
	if err != nil {
		log.Fatalln(err.Error())

		return Object{}
	}

	return p
}

// Update test
func (t *Resource) Update(p Object, id string) Object {
	// check item if exists
	payload := t.Get(id)
	if payload.ID == "" {
		log.Fatalln("not exists")

		return Object{}
	}

	// meta data
	p.UpdatedAt = time.Now()

	// insert to database
	_, err := r.Table(tableName).Get(id).Update(p).Run(module.RSession)
	if err != nil {
		log.Fatalln(err.Error())

		return Object{}
	}

	return p
}

// Remove test
func (t *Resource) Remove(id string) bool {
	// check item if exists
	payload := t.Get(id)
	if payload.ID == "" {
		log.Fatalln("not exists")

		return false
	}

	// insert to database
	_, err := r.Table(tableName).Get(id).Delete().Run(module.RSession)
	if err != nil {
		log.Fatalln(err.Error())

		return false
	}

	return true
}
<|endoftext|>


<commit_before>/**
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package order

import (
	"github.ibm.com/riethm/gopherlayer.git/datatypes"
	"github.ibm.com/riethm/gopherlayer.git/services"
	"github.ibm.com/riethm/gopherlayer.git/session"
)

// CheckBillingOrderStatus returns true if the status of the billing order for
// the provided product order receipt is in the list of provided statuses.
// Returns false otherwise, along with the billing order item used to check the statuses,
// and any error encountered.
func CheckBillingOrderStatus(sess *session.Session, receipt *datatypes.Container_Product_Order_Receipt, statuses []string) (bool, *datatypes.Billing_Order_Item, error) {
	service := services.GetBillingOrderItemService(sess)

	item, err := service.
		Id(*receipt.PlacedOrder.Items[0].Id).
		Mask("mask[id,billingItem[id,provisionTransaction[id,transactionStatus[name]]]]").
		GetObject()

	if err != nil {
		return false, err
	}

	currentStatus := *item.BillingItem.ProvisionTransaction.TransactionStatus.Name
	for _, status := range statuses {
		if currentStatus == status {
			return true, nil
		}
	}

	return false, item.BillingItem, nil
}

// CheckBillingOrderComplete returns true if the status of the billing order for
// the provided product order receipt is "COMPLETE". Returns false otherwise,
// along with the billing order item used to check the statuses, and any error encountered.
func CheckBillingOrderComplete(sess *session.Session, receipt *datatypes.Container_Product_Order_Receipt) (bool, *datatypes.Billing_Order_Item, error) {
	return CheckBillingOrderStatus(sess, receipt, []string{"COMPLETE"})
}
<commit_msg>Return biling order item<commit_after>/**
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package order

import (
	"github.ibm.com/riethm/gopherlayer.git/datatypes"
	"github.ibm.com/riethm/gopherlayer.git/services"
	"github.ibm.com/riethm/gopherlayer.git/session"
)

// CheckBillingOrderStatus returns true if the status of the billing order for
// the provided product order receipt is in the list of provided statuses.
// Returns false otherwise, along with the billing order item used to check the statuses,
// and any error encountered.
func CheckBillingOrderStatus(sess *session.Session, receipt *datatypes.Container_Product_Order_Receipt, statuses []string) (bool, *datatypes.Billing_Order_Item, error) {
	service := services.GetBillingOrderItemService(sess)

	item, err := service.
		Id(*receipt.PlacedOrder.Items[0].Id).
		Mask("mask[id,billingItem[id,provisionTransaction[id,transactionStatus[name]]]]").
		GetObject()

	if err != nil {
		return false, err
	}

	currentStatus := *item.BillingItem.ProvisionTransaction.TransactionStatus.Name
	for _, status := range statuses {
		if currentStatus == status {
			return true, nil
		}
	}

	return false, item, nil
}

// CheckBillingOrderComplete returns true if the status of the billing order for
// the provided product order receipt is "COMPLETE". Returns false otherwise,
// along with the billing order item used to check the statuses, and any error encountered.
func CheckBillingOrderComplete(sess *session.Session, receipt *datatypes.Container_Product_Order_Receipt) (bool, *datatypes.Billing_Order_Item, error) {
	return CheckBillingOrderStatus(sess, receipt, []string{"COMPLETE"})
}
<|endoftext|>


<commit_before>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import (
	"context"
	"testing"
	"time"

	"github.com/keybase/kbfs/kbfsblock"
	"github.com/keybase/kbfs/kbfscrypto"
	"github.com/keybase/kbfs/tlf"
	"github.com/stretchr/testify/require"
	"github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/storage"
	"github.com/syndtr/goleveldb/leveldb/util"
)

const (
	testDiskBlockCacheMaxBytes uint64 = 1 << 20
)

type testDiskBlockCacheConfig struct {
	codecGetter
	logMaker
	*testClockGetter
}

func newTestDiskBlockCacheConfig(t *testing.T) testDiskBlockCacheConfig {
	return testDiskBlockCacheConfig{
		newTestCodecGetter(),
		newTestLogMaker(t),
		newTestClockGetter(),
	}
}

func newDiskBlockCacheStandardForTest(config diskBlockCacheConfig,
	maxBytes uint64) (*DiskBlockCacheStandard, error) {
	blockStorage := storage.NewMemStorage()
	lruStorage := storage.NewMemStorage()
	tlfStorage := storage.NewMemStorage()
	return newDiskBlockCacheStandardFromStorage(config, blockStorage,
		lruStorage, tlfStorage, maxBytes)
}

func initDiskBlockCacheTest(t *testing.T) (*DiskBlockCacheStandard,
	testDiskBlockCacheConfig) {
	config := newTestDiskBlockCacheConfig(t)
	cache, err := newDiskBlockCacheStandardForTest(config,
		testDiskBlockCacheMaxBytes)
	require.NoError(t, err)
	return cache, config
}

func shutdownDiskBlockCacheTest(cache DiskBlockCache) {
	cache.Shutdown(context.Background())
}

func setupBlockForDiskCache(t *testing.T, config diskBlockCacheConfig) (
	kbfsblock.ID, []byte, kbfscrypto.BlockCryptKeyServerHalf) {
	ptr := makeRandomBlockPointer(t)
	block := makeFakeFileBlock(t, false)
	blockEncoded, err := config.Codec().Encode(block)
	require.NoError(t, err)
	serverHalf, err := kbfscrypto.MakeRandomBlockCryptKeyServerHalf()
	require.NoError(t, err)
	return ptr.ID, blockEncoded, serverHalf
}

func TestDiskBlockCachePutAndGet(t *testing.T) {
	t.Parallel()
	t.Log("Test that basic disk cache Put and Get operations work.")
	cache, config := initDiskBlockCacheTest(t)
	defer shutdownDiskBlockCacheTest(cache)

	tlf1 := tlf.FakeID(0, false)
	block1Id, block1Encoded, block1ServerHalf := setupBlockForDiskCache(t, config)

	ctx := context.Background()

	t.Log("Put a block into the cache.")
	err := cache.Put(ctx, tlf1, block1Id, block1Encoded, block1ServerHalf)
	require.NoError(t, err)
	putTime, err := cache.getLRU(block1Id.Bytes())
	require.NoError(t, err)
	config.TestClock().Add(time.Second)

	t.Log("Get that block from the cache. Verify that it's the same.")
	buf, serverHalf, err := cache.Get(ctx, tlf1, block1Id)
	require.NoError(t, err)
	require.Equal(t, block1ServerHalf, serverHalf)
	require.Equal(t, block1Encoded, buf)

	t.Log("Verify that the Get updated the LRU time for the block.")
	getTime, err := cache.getLRU(block1Id.Bytes())
	require.NoError(t, err)
	require.True(t, getTime.After(putTime))

	t.Log("Attempt to Get a block from the cache that isn't there." +
		" Verify that it fails.")
	ptr2 := makeRandomBlockPointer(t)
	buf, serverHalf, err = cache.Get(ctx, tlf1, ptr2.ID)
	require.EqualError(t, err, NoSuchBlockError{ptr2.ID}.Error())
	require.Equal(t, kbfscrypto.BlockCryptKeyServerHalf{}, serverHalf)
	require.Nil(t, buf)

	t.Log("Verify that the cache returns no LRU time for the missing block.")
	_, err = cache.getLRU(ptr2.ID.Bytes())
	require.EqualError(t, err, errors.ErrNotFound.Error())
}

func TestDiskBlockCacheDelete(t *testing.T) {
	t.Parallel()
	t.Log("Test that disk cache deletion works.")
	cache, config := initDiskBlockCacheTest(t)
	defer shutdownDiskBlockCacheTest(cache)
	ctx := context.Background()

	t.Log("Seed the cache with some other TLFs")
	fakeTlfs := []byte{0, 1, 2, 4, 5}
	for _, f := range fakeTlfs {
		tlf := tlf.FakeID(f, false)
		blockId, blockEncoded, serverHalf := setupBlockForDiskCache(t, config)
		err := cache.Put(ctx, tlf, blockId, blockEncoded, serverHalf)
		require.NoError(t, err)
	}
	tlf1 := tlf.FakeID(3, false)
	block1Id, block1Encoded, block1ServerHalf := setupBlockForDiskCache(t, config)
	block2Id, block2Encoded, block2ServerHalf := setupBlockForDiskCache(t, config)
	block3Id, block3Encoded, block3ServerHalf := setupBlockForDiskCache(t, config)

	t.Log("Put three blocks into the cache.")
	err := cache.Put(ctx, tlf1, block1Id, block1Encoded, block1ServerHalf)
	require.NoError(t, err)
	err = cache.Put(ctx, tlf1, block2Id, block2Encoded, block2ServerHalf)
	require.NoError(t, err)
	err = cache.Put(ctx, tlf1, block3Id, block3Encoded, block3ServerHalf)
	require.NoError(t, err)

	t.Log("Delete two of the blocks from the cache.")
	err = cache.DeleteByTLF(ctx, tlf1, []kbfsblock.ID{
		block1Id, block2Id})
	require.NoError(t, err)

	t.Log("Verify that only the non-deleted block is still in the cache.")
	_, _, err = cache.Get(ctx, tlf1, block1Id)
	require.EqualError(t, err, NoSuchBlockError{block1Id}.Error())
	_, _, err = cache.Get(ctx, tlf1, block2Id)
	require.EqualError(t, err, NoSuchBlockError{block2Id}.Error())
	_, _, err = cache.Get(ctx, tlf1, block3Id)
	require.NoError(t, err)

	t.Log("Verify that the cache returns no LRU time for the missing blocks.")
	_, err = cache.getLRU(block1Id.Bytes())
	require.EqualError(t, err, errors.ErrNotFound.Error())
	_, err = cache.getLRU(block2Id.Bytes())
	require.EqualError(t, err, errors.ErrNotFound.Error())
}

func TestDiskBlockCacheEvictFromTLF(t *testing.T) {
	t.Parallel()
	t.Log("Test that disk cache eviction works.")
	cache, config := initDiskBlockCacheTest(t)
	defer shutdownDiskBlockCacheTest(cache)

	tlf1 := tlf.FakeID(3, false)
	ctx := context.Background()
	clock := config.TestClock()
	initialTime := clock.Now()
	t.Log("Seed the cache with some other TLFs")
	//fakeTlfs := []byte{}
	fakeTlfs := []byte{0, 1, 2, 4, 5}
	for _, f := range fakeTlfs {
		tlf := tlf.FakeID(f, false)
		blockId, blockEncoded, serverHalf := setupBlockForDiskCache(t, config)
		err := cache.Put(ctx, tlf, blockId, blockEncoded, serverHalf)
		require.NoError(t, err)
		clock.Add(time.Second)
	}
	tlf1NumBlocks := 100
	t.Log("Put 100 blocks into the cache.")
	for i := 0; i < tlf1NumBlocks; i++ {
		blockId, blockEncoded, serverHalf := setupBlockForDiskCache(t, config)
		err := cache.Put(ctx, tlf1, blockId, blockEncoded, serverHalf)
		require.NoError(t, err)
		clock.Add(time.Second)
	}

	previousAvgDuration := 50 * time.Second
	averageDifference := float64(0)
	numEvictionDifferences := 0
	expectedCount := tlf1NumBlocks

	t.Log("Incrementally evict all the tlf1 blocks in the cache.")
	// Because the eviction algorithm is probabilistic, we can't rely on the
	// same number of blocks being evicted every time. So we have to be smart
	// about our assertions.
	for expectedCount != 0 {
		t.Log("Evict 10 blocks from the cache.")
		numRemoved, err := cache.evictFromTLFLocked(ctx, tlf1, 10)
		require.NoError(t, err)
		expectedCount -= numRemoved

		blockCount := 0
		var avgDuration time.Duration
		func() {
			tlfBytes := tlf1.Bytes()
			tlf1Range := util.BytesPrefix(tlfBytes)
			iter := cache.tlfDb.NewIterator(tlf1Range, nil)
			defer iter.Release()
			for iter.Next() {
				blockIDBytes := iter.Key()[len(tlfBytes):]
				putTime, err := cache.getLRU(blockIDBytes)
				require.NoError(t, err)
				avgDuration += putTime.Sub(initialTime)
				blockCount++
			}
		}()
		t.Logf("Verify that there are %d blocks in the cache.", expectedCount)
		require.Equal(t, expectedCount, blockCount,
			"Removed %d blocks this round.", numRemoved)
		if expectedCount > 0 {
			avgDuration /= time.Duration(expectedCount)
			t.Logf("Average LRU time of remaining blocks: %.2f",
				avgDuration.Seconds())
			averageDifference += avgDuration.Seconds() -
				previousAvgDuration.Seconds()
			previousAvgDuration = avgDuration
			numEvictionDifferences++
		}
	}
	t.Log("Verify that, on average, the LRU time of the blocks remaining in" +
		" the queue keeps going up.")
	averageDifference /= float64(numEvictionDifferences)
	require.True(t, averageDifference > 3.0,
		"Average overall LRU delta from an eviction: %.2f", averageDifference)
}
<commit_msg>disk_block_cache: Add test for overall LRU eviction.<commit_after>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libkbfs

import (
	"context"
	"testing"
	"time"

	"github.com/keybase/kbfs/kbfsblock"
	"github.com/keybase/kbfs/kbfscrypto"
	"github.com/keybase/kbfs/tlf"
	"github.com/stretchr/testify/require"
	"github.com/syndtr/goleveldb/leveldb/errors"
	"github.com/syndtr/goleveldb/leveldb/storage"
	"github.com/syndtr/goleveldb/leveldb/util"
)

const (
	testDiskBlockCacheMaxBytes uint64 = 1 << 20
)

type testDiskBlockCacheConfig struct {
	codecGetter
	logMaker
	*testClockGetter
}

func newTestDiskBlockCacheConfig(t *testing.T) testDiskBlockCacheConfig {
	return testDiskBlockCacheConfig{
		newTestCodecGetter(),
		newTestLogMaker(t),
		newTestClockGetter(),
	}
}

func newDiskBlockCacheStandardForTest(config diskBlockCacheConfig,
	maxBytes uint64) (*DiskBlockCacheStandard, error) {
	blockStorage := storage.NewMemStorage()
	lruStorage := storage.NewMemStorage()
	tlfStorage := storage.NewMemStorage()
	return newDiskBlockCacheStandardFromStorage(config, blockStorage,
		lruStorage, tlfStorage, maxBytes)
}

func initDiskBlockCacheTest(t *testing.T) (*DiskBlockCacheStandard,
	testDiskBlockCacheConfig) {
	config := newTestDiskBlockCacheConfig(t)
	cache, err := newDiskBlockCacheStandardForTest(config,
		testDiskBlockCacheMaxBytes)
	require.NoError(t, err)
	return cache, config
}

func shutdownDiskBlockCacheTest(cache DiskBlockCache) {
	cache.Shutdown(context.Background())
}

func setupBlockForDiskCache(t *testing.T, config diskBlockCacheConfig) (
	kbfsblock.ID, []byte, kbfscrypto.BlockCryptKeyServerHalf) {
	ptr := makeRandomBlockPointer(t)
	block := makeFakeFileBlock(t, false)
	blockEncoded, err := config.Codec().Encode(block)
	require.NoError(t, err)
	serverHalf, err := kbfscrypto.MakeRandomBlockCryptKeyServerHalf()
	require.NoError(t, err)
	return ptr.ID, blockEncoded, serverHalf
}

func TestDiskBlockCachePutAndGet(t *testing.T) {
	t.Parallel()
	t.Log("Test that basic disk cache Put and Get operations work.")
	cache, config := initDiskBlockCacheTest(t)
	defer shutdownDiskBlockCacheTest(cache)

	tlf1 := tlf.FakeID(0, false)
	block1Id, block1Encoded, block1ServerHalf := setupBlockForDiskCache(t, config)

	ctx := context.Background()

	t.Log("Put a block into the cache.")
	err := cache.Put(ctx, tlf1, block1Id, block1Encoded, block1ServerHalf)
	require.NoError(t, err)
	putTime, err := cache.getLRU(block1Id.Bytes())
	require.NoError(t, err)
	config.TestClock().Add(time.Second)

	t.Log("Get that block from the cache. Verify that it's the same.")
	buf, serverHalf, err := cache.Get(ctx, tlf1, block1Id)
	require.NoError(t, err)
	require.Equal(t, block1ServerHalf, serverHalf)
	require.Equal(t, block1Encoded, buf)

	t.Log("Verify that the Get updated the LRU time for the block.")
	getTime, err := cache.getLRU(block1Id.Bytes())
	require.NoError(t, err)
	require.True(t, getTime.After(putTime))

	t.Log("Attempt to Get a block from the cache that isn't there." +
		" Verify that it fails.")
	ptr2 := makeRandomBlockPointer(t)
	buf, serverHalf, err = cache.Get(ctx, tlf1, ptr2.ID)
	require.EqualError(t, err, NoSuchBlockError{ptr2.ID}.Error())
	require.Equal(t, kbfscrypto.BlockCryptKeyServerHalf{}, serverHalf)
	require.Nil(t, buf)

	t.Log("Verify that the cache returns no LRU time for the missing block.")
	_, err = cache.getLRU(ptr2.ID.Bytes())
	require.EqualError(t, err, errors.ErrNotFound.Error())
}

func TestDiskBlockCacheDelete(t *testing.T) {
	t.Parallel()
	t.Log("Test that disk cache deletion works.")
	cache, config := initDiskBlockCacheTest(t)
	defer shutdownDiskBlockCacheTest(cache)
	ctx := context.Background()

	t.Log("Seed the cache with some other TLFs")
	fakeTlfs := []byte{0, 1, 2, 4, 5}
	for _, f := range fakeTlfs {
		tlf := tlf.FakeID(f, false)
		blockId, blockEncoded, serverHalf := setupBlockForDiskCache(t, config)
		err := cache.Put(ctx, tlf, blockId, blockEncoded, serverHalf)
		require.NoError(t, err)
	}
	tlf1 := tlf.FakeID(3, false)
	block1Id, block1Encoded, block1ServerHalf := setupBlockForDiskCache(t, config)
	block2Id, block2Encoded, block2ServerHalf := setupBlockForDiskCache(t, config)
	block3Id, block3Encoded, block3ServerHalf := setupBlockForDiskCache(t, config)

	t.Log("Put three blocks into the cache.")
	err := cache.Put(ctx, tlf1, block1Id, block1Encoded, block1ServerHalf)
	require.NoError(t, err)
	err = cache.Put(ctx, tlf1, block2Id, block2Encoded, block2ServerHalf)
	require.NoError(t, err)
	err = cache.Put(ctx, tlf1, block3Id, block3Encoded, block3ServerHalf)
	require.NoError(t, err)

	t.Log("Delete two of the blocks from the cache.")
	err = cache.DeleteByTLF(ctx, tlf1, []kbfsblock.ID{
		block1Id, block2Id})
	require.NoError(t, err)

	t.Log("Verify that only the non-deleted block is still in the cache.")
	_, _, err = cache.Get(ctx, tlf1, block1Id)
	require.EqualError(t, err, NoSuchBlockError{block1Id}.Error())
	_, _, err = cache.Get(ctx, tlf1, block2Id)
	require.EqualError(t, err, NoSuchBlockError{block2Id}.Error())
	_, _, err = cache.Get(ctx, tlf1, block3Id)
	require.NoError(t, err)

	t.Log("Verify that the cache returns no LRU time for the missing blocks.")
	_, err = cache.getLRU(block1Id.Bytes())
	require.EqualError(t, err, errors.ErrNotFound.Error())
	_, err = cache.getLRU(block2Id.Bytes())
	require.EqualError(t, err, errors.ErrNotFound.Error())
}

func TestDiskBlockCacheEvictFromTLF(t *testing.T) {
	t.Parallel()
	t.Log("Test that disk cache eviction works for a single TLF.")
	cache, config := initDiskBlockCacheTest(t)
	defer shutdownDiskBlockCacheTest(cache)

	tlf1 := tlf.FakeID(3, false)
	ctx := context.Background()
	clock := config.TestClock()
	initialTime := clock.Now()
	t.Log("Seed the cache with some other TLFs")
	//fakeTlfs := []byte{}
	fakeTlfs := []byte{0, 1, 2, 4, 5}
	for _, f := range fakeTlfs {
		tlf := tlf.FakeID(f, false)
		blockId, blockEncoded, serverHalf := setupBlockForDiskCache(t, config)
		err := cache.Put(ctx, tlf, blockId, blockEncoded, serverHalf)
		require.NoError(t, err)
		clock.Add(time.Second)
	}
	tlf1NumBlocks := 100
	t.Log("Put 100 blocks into the cache.")
	for i := 0; i < tlf1NumBlocks; i++ {
		blockId, blockEncoded, serverHalf := setupBlockForDiskCache(t, config)
		err := cache.Put(ctx, tlf1, blockId, blockEncoded, serverHalf)
		require.NoError(t, err)
		clock.Add(time.Second)
	}

	previousAvgDuration := 50 * time.Second
	averageDifference := float64(0)
	numEvictionDifferences := 0
	expectedCount := tlf1NumBlocks

	t.Log("Incrementally evict all the tlf1 blocks in the cache.")
	// Because the eviction algorithm is probabilistic, we can't rely on the
	// same number of blocks being evicted every time. So we have to be smart
	// about our assertions.
	for expectedCount != 0 {
		t.Log("Evict 10 blocks from the cache.")
		numRemoved, err := cache.evictFromTLFLocked(ctx, tlf1, 10)
		require.NoError(t, err)
		expectedCount -= numRemoved

		blockCount := 0
		var avgDuration time.Duration
		func() {
			tlfBytes := tlf1.Bytes()
			tlf1Range := util.BytesPrefix(tlfBytes)
			iter := cache.tlfDb.NewIterator(tlf1Range, nil)
			defer iter.Release()
			for iter.Next() {
				blockIDBytes := iter.Key()[len(tlfBytes):]
				putTime, err := cache.getLRU(blockIDBytes)
				require.NoError(t, err)
				avgDuration += putTime.Sub(initialTime)
				blockCount++
			}
		}()
		t.Logf("Verify that there are %d blocks in the cache.", expectedCount)
		require.Equal(t, expectedCount, blockCount,
			"Removed %d blocks this round.", numRemoved)
		if expectedCount > 0 {
			avgDuration /= time.Duration(expectedCount)
			t.Logf("Average LRU time of remaining blocks: %.2f",
				avgDuration.Seconds())
			averageDifference += avgDuration.Seconds() -
				previousAvgDuration.Seconds()
			previousAvgDuration = avgDuration
			numEvictionDifferences++
		}
	}
	t.Log("Verify that, on average, the LRU time of the blocks remaining in" +
		" the queue keeps going up.")
	averageDifference /= float64(numEvictionDifferences)
	require.True(t, averageDifference > 3.0,
		"Average overall LRU delta from an eviction: %.2f", averageDifference)
}

func TestDiskBlockCacheEvictOverall(t *testing.T) {
	t.Parallel()
	t.Log("Test that disk cache eviction works overall.")
	cache, config := initDiskBlockCacheTest(t)
	defer shutdownDiskBlockCacheTest(cache)

	ctx := context.Background()
	clock := config.TestClock()
	initialTime := clock.Now()

	numTlfs := 10
	numBlocksPerTlf := 10
	totalBlocks := numTlfs * numBlocksPerTlf

	t.Log("Seed the cache with some other TLFs")
	for i := byte(0); int(i) < numTlfs; i++ {
		currTlf := tlf.FakeID(i, false)
		for j := 0; j < numBlocksPerTlf; j++ {
			blockId, blockEncoded, serverHalf := setupBlockForDiskCache(t, config)
			err := cache.Put(ctx, currTlf, blockId, blockEncoded, serverHalf)
			require.NoError(t, err)
			clock.Add(time.Second)
		}
	}

	// Average LRU will initially be half the total number of blocks, in
	// seconds.
	previousAvgDuration := time.Duration(totalBlocks>>1) * time.Second
	averageDifference := float64(0)
	numEvictionDifferences := 0
	expectedCount := totalBlocks

	t.Log("Incrementally evict all the blocks in the cache.")
	// Because the eviction algorithm is probabilistic, we can't rely on the
	// same number of blocks being evicted every time. So we have to be smart
	// about our assertions.
	for expectedCount != 0 {
		t.Log("Evict 10 blocks from the cache.")
		numRemoved, err := cache.evictLocked(ctx, 10)
		require.NoError(t, err)
		expectedCount -= numRemoved

		blockCount := 0
		var avgDuration time.Duration
		func() {
			iter := cache.metaDb.NewIterator(nil, nil)
			defer iter.Release()
			for iter.Next() {
				metadata := diskBlockCacheMetadata{}
				err = config.Codec().Decode(iter.Value(), &metadata)
				require.NoError(t, err)
				avgDuration += metadata.LRUTime.Sub(initialTime)
				blockCount++
			}
		}()
		t.Logf("Verify that there are %d blocks in the cache.", expectedCount)
		require.Equal(t, expectedCount, blockCount,
			"Removed %d blocks this round.", numRemoved)
		if expectedCount > 0 {
			avgDuration /= time.Duration(expectedCount)
			t.Logf("Average LRU time of remaining blocks: %.2f",
				avgDuration.Seconds())
			averageDifference += avgDuration.Seconds() -
				previousAvgDuration.Seconds()
			previousAvgDuration = avgDuration
			numEvictionDifferences++
		}
	}
	t.Log("Verify that, on average, the LRU time of the blocks remaining in" +
		" the queue keeps going up.")
	averageDifference /= float64(numEvictionDifferences)
	require.True(t, averageDifference > 3.0,
		"Average overall LRU delta from an eviction: %.2f", averageDifference)
}
<|endoftext|>


<commit_before>package models

import (
	"appengine"
	"appengine/datastore"
	"fmt"
	"net/url"
	"time"
)

type Mention struct {
	Source   url.URL
	Target   url.URL
	Recieved time.Time
	Verified bool
}

func NewMention(c appengine.Context, source string, target string) (*Mention, error) {
	var k *datastore.Key
	var e *Mention

	k = GetMention(c, source, target)
	if k == nil {
		k = datastore.NewIncompleteKey(c, "Mention", nil)
		e = new(Mention)
		src, err := url.Parse(source)
		if err != nil {
			return nil, err
		}

		trg, err := url.Parse(target)
		if err != nil {
			return nil, err
		}

		e.Source = src
		e.target = trg
		e.Recieved = time.Now()
		e.Verified = false

		_, err := datastore.Put(c, k, e)
		if err != nil {
			return nil, err
		}
	} else {
		if err = datastore.Get(c, k, &e); err != nil {
			return nil, err
		}
	}

	return e, nil
}

func MentionExists(c appengine.Context, source string, target string) bool {
	return GetMention(c, source, target) != nil
}

func GetMention(c appengine.Context, source string, target string) *datastore.Key {
	q := datastore.NewQuery("Mention").Filter("source =", source).Filter("target =", target).KeysOnly()
	k, err := q.Run(c).Next(nil)
	if err != nil {
		c.Infof("Error getting Mention (%s => %s): %v", source, target, err)
		return nil
	}
	return k
}
<commit_msg>bugs in mention<commit_after>package models

import (
	"appengine"
	"appengine/datastore"
	"net/url"
	"time"
)

type Mention struct {
	Source   url.URL
	Target   url.URL
	Recieved time.Time
	Verified bool
}

func NewMention(c appengine.Context, source string, target string) (*Mention, error) {
	var k *datastore.Key
	var e *Mention

	k = GetMention(c, source, target)
	if k == nil {
		k = datastore.NewIncompleteKey(c, "Mention", nil)
		e = new(Mention)
		src, err := url.Parse(source)
		if err != nil {
			return nil, err
		}

		trg, err := url.Parse(target)
		if err != nil {
			return nil, err
		}

		e.Source = &src
		e.Target = &trg
		e.Recieved = time.Now()
		e.Verified = false

		k, err = datastore.Put(c, k, e)
		if err != nil {
			return nil, err
		}
	} else {
		if err = datastore.Get(c, k, &e); err != nil {
			return nil, err
		}
	}

	return e, nil
}

func MentionExists(c appengine.Context, source string, target string) bool {
	return GetMention(c, source, target) != nil
}

func GetMention(c appengine.Context, source string, target string) *datastore.Key {
	q := datastore.NewQuery("Mention").Filter("source =", source).Filter("target =", target).KeysOnly()
	k, err := q.Run(c).Next(nil)
	if err != nil {
		c.Infof("Error getting Mention (%s => %s): %v", source, target, err)
		return nil
	}
	return k
}
<|endoftext|>


<commit_before>package vcs

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
)

type hg struct {
	cmd string
}

func (_ hg) ShortName() string { return "hg" }

var Hg VCS = hg{"hg"}

type hgRepo struct {
	dir string
	hg  *hg
}

func (hg hg) Clone(url, dir string) (Repository, error) {
	r := &hgRepo{dir, &hg}

	cmd := exec.Command("hg", "clone", "--", url, dir)
	if out, err := cmd.CombinedOutput(); err != nil {
		if strings.Contains(string(out), fmt.Sprintf("abort: destination '%s' is not empty", dir)) {
			return nil, os.ErrExist
		}
		return nil, fmt.Errorf("hg clone failed: %s\n%s", err, out)
	}

	return r, nil
}

func (hg hg) Open(dir string) (Repository, error) {
	// TODO(sqs): check for .hg or bare repo
	if _, err := os.Stat(dir); err == nil {
		return &hgRepo{dir, &hg}, nil
	} else {
		return nil, err
	}
}

func (hg hg) CloneMirror(url, dir string) error {
	panic("not yet implemented")
}

func (hg hg) UpdateMirror(dir string) error {
	panic("not yet implemented")
}

func (r *hgRepo) Dir() (dir string) {
	return r.dir
}

func (r *hgRepo) VCS() VCS {
	return r.hg
}

func (r *hgRepo) Download() error {
	cmd := exec.Command("hg", "pull")
	cmd.Dir = r.dir
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("hg pull failed: %s\n%s", err, out)
	}
	return nil
}

func (r *hgRepo) CheckOut(rev string) (dir string, err error) {
	if rev == "" {
		rev = "default"
	}
	cmd := exec.Command("hg", "update", "-r", rev)
	cmd.Dir = r.dir
	if out, err := cmd.CombinedOutput(); err == nil {
		return r.dir, nil
	} else {
		return "", fmt.Errorf("hg update -r %q failed: %s\n%s", rev, err, out)
	}
}
<commit_msg>hg {Clone,Update}Mirror<commit_after>package vcs

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
)

type hg struct {
	cmd string
}

func (_ hg) ShortName() string { return "hg" }

var Hg VCS = hg{"hg"}

type hgRepo struct {
	dir string
	hg  *hg
}

func (hg hg) Clone(url, dir string) (Repository, error) {
	r := &hgRepo{dir, &hg}

	cmd := exec.Command("hg", "clone", "--", url, dir)
	if out, err := cmd.CombinedOutput(); err != nil {
		if strings.Contains(string(out), fmt.Sprintf("abort: destination '%s' is not empty", dir)) {
			return nil, os.ErrExist
		}
		return nil, fmt.Errorf("hg clone failed: %s\n%s", err, out)
	}

	return r, nil
}

func (hg hg) Open(dir string) (Repository, error) {
	// TODO(sqs): check for .hg or bare repo
	if _, err := os.Stat(dir); err == nil {
		return &hgRepo{dir, &hg}, nil
	} else {
		return nil, err
	}
}

func (hg hg) CloneMirror(url, dir string) error {
	cmd := exec.Command("hg", "clone", "-U", "--", url, dir)
	if out, err := cmd.CombinedOutput(); err != nil {
		if strings.Contains(string(out), fmt.Sprintf("abort: destination '%s' is not empty", dir)) {
			return os.ErrExist
		}
		return fmt.Errorf("hg %v failed: %s\n%s", cmd.Args, err, out)
	}
	return nil
}

func (hg hg) UpdateMirror(dir string) error {
	cmd := exec.Command("hg", "pull")
	cmd.Dir = dir
	if out, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("hg %v failed: %s\n%s", cmd.Args, err, out)
	}
	return nil
}

func (r *hgRepo) Dir() (dir string) {
	return r.dir
}

func (r *hgRepo) VCS() VCS {
	return r.hg
}

func (r *hgRepo) Download() error {
	cmd := exec.Command("hg", "pull")
	cmd.Dir = r.dir
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("hg pull failed: %s\n%s", err, out)
	}
	return nil
}

func (r *hgRepo) CheckOut(rev string) (dir string, err error) {
	if rev == "" {
		rev = "default"
	}
	cmd := exec.Command("hg", "update", "-r", rev)
	cmd.Dir = r.dir
	if out, err := cmd.CombinedOutput(); err == nil {
		return r.dir, nil
	} else {
		return "", fmt.Errorf("hg update -r %q failed: %s\n%s", rev, err, out)
	}
}
<|endoftext|>


<commit_before>package networkdb

import (
	"fmt"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/gogo/protobuf/proto"
)

type delegate struct {
	nDB *NetworkDB
}

func (d *delegate) NodeMeta(limit int) []byte {
	return []byte{}
}

func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool {
	// Update our local clock if the received messages has newer
	// time.
	nDB.networkClock.Witness(nEvent.LTime)

	nDB.Lock()
	defer nDB.Unlock()

	nodeNetworks, ok := nDB.networks[nEvent.NodeName]
	if !ok {
		// We haven't heard about this node at all.  Ignore the leave
		if nEvent.Type == NetworkEventTypeLeave {
			return false
		}

		nodeNetworks = make(map[string]*network)
		nDB.networks[nEvent.NodeName] = nodeNetworks
	}

	if n, ok := nodeNetworks[nEvent.NetworkID]; ok {
		// We have the latest state. Ignore the event
		// since it is stale.
		if n.ltime >= nEvent.LTime {
			return false
		}

		n.ltime = nEvent.LTime
		n.leaving = nEvent.Type == NetworkEventTypeLeave
		if n.leaving {
			n.leaveTime = time.Now()
		}

		return true
	}

	if nEvent.Type == NetworkEventTypeLeave {
		return false
	}

	// This remote network join is being seen the first time.
	nodeNetworks[nEvent.NetworkID] = &network{
		id:    nEvent.NetworkID,
		ltime: nEvent.LTime,
	}

	nDB.networkNodes[nEvent.NetworkID] = append(nDB.networkNodes[nEvent.NetworkID], nEvent.NodeName)
	return true
}

func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool {
	// Update our local clock if the received messages has newer
	// time.
	nDB.tableClock.Witness(tEvent.LTime)

	if entry, err := nDB.getEntry(tEvent.TableName, tEvent.NetworkID, tEvent.Key); err == nil {
		// We have the latest state. Ignore the event
		// since it is stale.
		if entry.ltime >= tEvent.LTime {
			return false
		}
	}

	entry := &entry{
		ltime:    tEvent.LTime,
		node:     tEvent.NodeName,
		value:    tEvent.Value,
		deleting: tEvent.Type == TableEventTypeDelete,
	}

	if entry.deleting {
		entry.deleteTime = time.Now()
	}

	nDB.Lock()
	nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tEvent.TableName, tEvent.NetworkID, tEvent.Key), entry)
	nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", tEvent.NetworkID, tEvent.TableName, tEvent.Key), entry)
	nDB.Unlock()

	var op opType
	switch tEvent.Type {
	case TableEventTypeCreate:
		op = opCreate
	case TableEventTypeUpdate:
		op = opUpdate
	case TableEventTypeDelete:
		op = opDelete
	}

	nDB.broadcaster.Write(makeEvent(op, tEvent.TableName, tEvent.NetworkID, tEvent.Key, tEvent.Value))
	return true
}

func (nDB *NetworkDB) handleCompound(buf []byte, isBulkSync bool) {
	// Decode the parts
	parts, err := decodeCompoundMessage(buf)
	if err != nil {
		logrus.Errorf("Failed to decode compound request: %v", err)
		return
	}

	// Handle each message
	for _, part := range parts {
		nDB.handleMessage(part, isBulkSync)
	}
}

func (nDB *NetworkDB) handleTableMessage(buf []byte, isBulkSync bool) {
	var tEvent TableEvent
	if err := proto.Unmarshal(buf, &tEvent); err != nil {
		logrus.Errorf("Error decoding table event message: %v", err)
		return
	}

	// Do not rebroadcast a bulk sync
	if rebroadcast := nDB.handleTableEvent(&tEvent); rebroadcast && !isBulkSync {
		var err error
		buf, err = encodeRawMessage(MessageTypeTableEvent, buf)
		if err != nil {
			logrus.Errorf("Error marshalling gossip message for network event rebroadcast: %v", err)
			return
		}

		nDB.RLock()
		n, ok := nDB.networks[nDB.config.NodeName][tEvent.NetworkID]
		nDB.RUnlock()

		if !ok {
			return
		}

		broadcastQ := n.tableBroadcasts

		if broadcastQ == nil {
			return
		}

		broadcastQ.QueueBroadcast(&tableEventMessage{
			msg:   buf,
			id:    tEvent.NetworkID,
			tname: tEvent.TableName,
			key:   tEvent.Key,
			node:  nDB.config.NodeName,
		})
	}
}

func (nDB *NetworkDB) handleNetworkMessage(buf []byte) {
	var nEvent NetworkEvent
	if err := proto.Unmarshal(buf, &nEvent); err != nil {
		logrus.Errorf("Error decoding network event message: %v", err)
		return
	}

	if rebroadcast := nDB.handleNetworkEvent(&nEvent); rebroadcast {
		var err error
		buf, err = encodeRawMessage(MessageTypeNetworkEvent, buf)
		if err != nil {
			logrus.Errorf("Error marshalling gossip message for network event rebroadcast: %v", err)
			return
		}

		nDB.networkBroadcasts.QueueBroadcast(&networkEventMessage{
			msg:  buf,
			id:   nEvent.NetworkID,
			node: nEvent.NodeName,
		})
	}
}

func (nDB *NetworkDB) handleBulkSync(buf []byte) {
	var bsm BulkSyncMessage
	if err := proto.Unmarshal(buf, &bsm); err != nil {
		logrus.Errorf("Error decoding bulk sync message: %v", err)
		return
	}

	if bsm.LTime > 0 {
		nDB.tableClock.Witness(bsm.LTime)
	}

	nDB.handleMessage(bsm.Payload, true)

	// Don't respond to a bulk sync which was not unsolicited
	if !bsm.Unsolicited {
		nDB.RLock()
		ch, ok := nDB.bulkSyncAckTbl[bsm.NodeName]
		nDB.RUnlock()
		if ok {
			close(ch)
		}

		return
	}

	if err := nDB.bulkSyncNode(bsm.Networks, bsm.NodeName, false); err != nil {
		logrus.Errorf("Error in responding to bulk sync from node %s: %v", nDB.nodes[bsm.NodeName].Addr, err)
	}
}

func (nDB *NetworkDB) handleMessage(buf []byte, isBulkSync bool) {
	mType, data, err := decodeMessage(buf)
	if err != nil {
		logrus.Errorf("Error decoding gossip message to get message type: %v", err)
		return
	}

	switch mType {
	case MessageTypeNetworkEvent:
		nDB.handleNetworkMessage(data)
	case MessageTypeTableEvent:
		nDB.handleTableMessage(data, isBulkSync)
	case MessageTypeBulkSync:
		nDB.handleBulkSync(data)
	case MessageTypeCompound:
		nDB.handleCompound(data, isBulkSync)
	default:
		logrus.Errorf("%s: unknown message type %d", nDB.config.NodeName, mType)
	}
}

func (d *delegate) NotifyMsg(buf []byte) {
	if len(buf) == 0 {
		return
	}

	d.nDB.handleMessage(buf, false)
}

func (d *delegate) GetBroadcasts(overhead, limit int) [][]byte {
	return d.nDB.networkBroadcasts.GetBroadcasts(overhead, limit)
}

func (d *delegate) LocalState(join bool) []byte {
	d.nDB.RLock()
	defer d.nDB.RUnlock()

	pp := NetworkPushPull{
		LTime: d.nDB.networkClock.Time(),
	}

	for name, nn := range d.nDB.networks {
		for _, n := range nn {
			pp.Networks = append(pp.Networks, &NetworkEntry{
				LTime:     n.ltime,
				NetworkID: n.id,
				NodeName:  name,
				Leaving:   n.leaving,
			})
		}
	}

	buf, err := encodeMessage(MessageTypePushPull, &pp)
	if err != nil {
		logrus.Errorf("Failed to encode local network state: %v", err)
		return nil
	}

	return buf
}

func (d *delegate) MergeRemoteState(buf []byte, isJoin bool) {
	if len(buf) == 0 {
		logrus.Error("zero byte remote network state received")
		return
	}

	var gMsg GossipMessage
	err := proto.Unmarshal(buf, &gMsg)
	if err != nil {
		logrus.Errorf("Error unmarshalling push pull messsage: %v", err)
		return
	}

	if gMsg.Type != MessageTypePushPull {
		logrus.Errorf("Invalid message type %v received from remote", buf[0])
	}

	pp := NetworkPushPull{}
	if err := proto.Unmarshal(gMsg.Data, &pp); err != nil {
		logrus.Errorf("Failed to decode remote network state: %v", err)
		return
	}

	if pp.LTime > 0 {
		d.nDB.networkClock.Witness(pp.LTime)
	}

	for _, n := range pp.Networks {
		nEvent := &NetworkEvent{
			LTime:     n.LTime,
			NodeName:  n.NodeName,
			NetworkID: n.NetworkID,
			Type:      NetworkEventTypeJoin,
		}

		if n.Leaving {
			nEvent.Type = NetworkEventTypeLeave
		}

		d.nDB.handleNetworkEvent(nEvent)
	}

}
<commit_msg>Make sure node map is valid before accessing it<commit_after>package networkdb

import (
	"fmt"
	"net"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/gogo/protobuf/proto"
)

type delegate struct {
	nDB *NetworkDB
}

func (d *delegate) NodeMeta(limit int) []byte {
	return []byte{}
}

func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool {
	// Update our local clock if the received messages has newer
	// time.
	nDB.networkClock.Witness(nEvent.LTime)

	nDB.Lock()
	defer nDB.Unlock()

	nodeNetworks, ok := nDB.networks[nEvent.NodeName]
	if !ok {
		// We haven't heard about this node at all.  Ignore the leave
		if nEvent.Type == NetworkEventTypeLeave {
			return false
		}

		nodeNetworks = make(map[string]*network)
		nDB.networks[nEvent.NodeName] = nodeNetworks
	}

	if n, ok := nodeNetworks[nEvent.NetworkID]; ok {
		// We have the latest state. Ignore the event
		// since it is stale.
		if n.ltime >= nEvent.LTime {
			return false
		}

		n.ltime = nEvent.LTime
		n.leaving = nEvent.Type == NetworkEventTypeLeave
		if n.leaving {
			n.leaveTime = time.Now()
		}

		return true
	}

	if nEvent.Type == NetworkEventTypeLeave {
		return false
	}

	// This remote network join is being seen the first time.
	nodeNetworks[nEvent.NetworkID] = &network{
		id:    nEvent.NetworkID,
		ltime: nEvent.LTime,
	}

	nDB.networkNodes[nEvent.NetworkID] = append(nDB.networkNodes[nEvent.NetworkID], nEvent.NodeName)
	return true
}

func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool {
	// Update our local clock if the received messages has newer
	// time.
	nDB.tableClock.Witness(tEvent.LTime)

	if entry, err := nDB.getEntry(tEvent.TableName, tEvent.NetworkID, tEvent.Key); err == nil {
		// We have the latest state. Ignore the event
		// since it is stale.
		if entry.ltime >= tEvent.LTime {
			return false
		}
	}

	entry := &entry{
		ltime:    tEvent.LTime,
		node:     tEvent.NodeName,
		value:    tEvent.Value,
		deleting: tEvent.Type == TableEventTypeDelete,
	}

	if entry.deleting {
		entry.deleteTime = time.Now()
	}

	nDB.Lock()
	nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tEvent.TableName, tEvent.NetworkID, tEvent.Key), entry)
	nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", tEvent.NetworkID, tEvent.TableName, tEvent.Key), entry)
	nDB.Unlock()

	var op opType
	switch tEvent.Type {
	case TableEventTypeCreate:
		op = opCreate
	case TableEventTypeUpdate:
		op = opUpdate
	case TableEventTypeDelete:
		op = opDelete
	}

	nDB.broadcaster.Write(makeEvent(op, tEvent.TableName, tEvent.NetworkID, tEvent.Key, tEvent.Value))
	return true
}

func (nDB *NetworkDB) handleCompound(buf []byte, isBulkSync bool) {
	// Decode the parts
	parts, err := decodeCompoundMessage(buf)
	if err != nil {
		logrus.Errorf("Failed to decode compound request: %v", err)
		return
	}

	// Handle each message
	for _, part := range parts {
		nDB.handleMessage(part, isBulkSync)
	}
}

func (nDB *NetworkDB) handleTableMessage(buf []byte, isBulkSync bool) {
	var tEvent TableEvent
	if err := proto.Unmarshal(buf, &tEvent); err != nil {
		logrus.Errorf("Error decoding table event message: %v", err)
		return
	}

	// Do not rebroadcast a bulk sync
	if rebroadcast := nDB.handleTableEvent(&tEvent); rebroadcast && !isBulkSync {
		var err error
		buf, err = encodeRawMessage(MessageTypeTableEvent, buf)
		if err != nil {
			logrus.Errorf("Error marshalling gossip message for network event rebroadcast: %v", err)
			return
		}

		nDB.RLock()
		n, ok := nDB.networks[nDB.config.NodeName][tEvent.NetworkID]
		nDB.RUnlock()

		if !ok {
			return
		}

		broadcastQ := n.tableBroadcasts

		if broadcastQ == nil {
			return
		}

		broadcastQ.QueueBroadcast(&tableEventMessage{
			msg:   buf,
			id:    tEvent.NetworkID,
			tname: tEvent.TableName,
			key:   tEvent.Key,
			node:  nDB.config.NodeName,
		})
	}
}

func (nDB *NetworkDB) handleNetworkMessage(buf []byte) {
	var nEvent NetworkEvent
	if err := proto.Unmarshal(buf, &nEvent); err != nil {
		logrus.Errorf("Error decoding network event message: %v", err)
		return
	}

	if rebroadcast := nDB.handleNetworkEvent(&nEvent); rebroadcast {
		var err error
		buf, err = encodeRawMessage(MessageTypeNetworkEvent, buf)
		if err != nil {
			logrus.Errorf("Error marshalling gossip message for network event rebroadcast: %v", err)
			return
		}

		nDB.networkBroadcasts.QueueBroadcast(&networkEventMessage{
			msg:  buf,
			id:   nEvent.NetworkID,
			node: nEvent.NodeName,
		})
	}
}

func (nDB *NetworkDB) handleBulkSync(buf []byte) {
	var bsm BulkSyncMessage
	if err := proto.Unmarshal(buf, &bsm); err != nil {
		logrus.Errorf("Error decoding bulk sync message: %v", err)
		return
	}

	if bsm.LTime > 0 {
		nDB.tableClock.Witness(bsm.LTime)
	}

	nDB.handleMessage(bsm.Payload, true)

	// Don't respond to a bulk sync which was not unsolicited
	if !bsm.Unsolicited {
		nDB.RLock()
		ch, ok := nDB.bulkSyncAckTbl[bsm.NodeName]
		nDB.RUnlock()
		if ok {
			close(ch)
		}

		return
	}

	var nodeAddr net.IP
	if node, ok := nDB.nodes[bsm.NodeName]; ok {
		nodeAddr = node.Addr
	}

	if err := nDB.bulkSyncNode(bsm.Networks, bsm.NodeName, false); err != nil {
		logrus.Errorf("Error in responding to bulk sync from node %s: %v", nodeAddr, err)
	}
}

func (nDB *NetworkDB) handleMessage(buf []byte, isBulkSync bool) {
	mType, data, err := decodeMessage(buf)
	if err != nil {
		logrus.Errorf("Error decoding gossip message to get message type: %v", err)
		return
	}

	switch mType {
	case MessageTypeNetworkEvent:
		nDB.handleNetworkMessage(data)
	case MessageTypeTableEvent:
		nDB.handleTableMessage(data, isBulkSync)
	case MessageTypeBulkSync:
		nDB.handleBulkSync(data)
	case MessageTypeCompound:
		nDB.handleCompound(data, isBulkSync)
	default:
		logrus.Errorf("%s: unknown message type %d", nDB.config.NodeName, mType)
	}
}

func (d *delegate) NotifyMsg(buf []byte) {
	if len(buf) == 0 {
		return
	}

	d.nDB.handleMessage(buf, false)
}

func (d *delegate) GetBroadcasts(overhead, limit int) [][]byte {
	return d.nDB.networkBroadcasts.GetBroadcasts(overhead, limit)
}

func (d *delegate) LocalState(join bool) []byte {
	d.nDB.RLock()
	defer d.nDB.RUnlock()

	pp := NetworkPushPull{
		LTime: d.nDB.networkClock.Time(),
	}

	for name, nn := range d.nDB.networks {
		for _, n := range nn {
			pp.Networks = append(pp.Networks, &NetworkEntry{
				LTime:     n.ltime,
				NetworkID: n.id,
				NodeName:  name,
				Leaving:   n.leaving,
			})
		}
	}

	buf, err := encodeMessage(MessageTypePushPull, &pp)
	if err != nil {
		logrus.Errorf("Failed to encode local network state: %v", err)
		return nil
	}

	return buf
}

func (d *delegate) MergeRemoteState(buf []byte, isJoin bool) {
	if len(buf) == 0 {
		logrus.Error("zero byte remote network state received")
		return
	}

	var gMsg GossipMessage
	err := proto.Unmarshal(buf, &gMsg)
	if err != nil {
		logrus.Errorf("Error unmarshalling push pull messsage: %v", err)
		return
	}

	if gMsg.Type != MessageTypePushPull {
		logrus.Errorf("Invalid message type %v received from remote", buf[0])
	}

	pp := NetworkPushPull{}
	if err := proto.Unmarshal(gMsg.Data, &pp); err != nil {
		logrus.Errorf("Failed to decode remote network state: %v", err)
		return
	}

	if pp.LTime > 0 {
		d.nDB.networkClock.Witness(pp.LTime)
	}

	for _, n := range pp.Networks {
		nEvent := &NetworkEvent{
			LTime:     n.LTime,
			NodeName:  n.NodeName,
			NetworkID: n.NetworkID,
			Type:      NetworkEventTypeJoin,
		}

		if n.Leaving {
			nEvent.Type = NetworkEventTypeLeave
		}

		d.nDB.handleNetworkEvent(nEvent)
	}

}
<|endoftext|>


<commit_before>// Package xid is a globally unique id generator suited for web scale
//
// Xid is using Mongo Object ID algorithm to generate globally unique ids:
// https://docs.mongodb.org/manual/reference/object-id/
//
//   - 4-byte value representing the seconds since the Unix epoch,
//   - 3-byte machine identifier,
//   - 2-byte process id, and
//   - 3-byte counter, starting with a random value.
//
// The binary representation of the id is compatible with Mongo 12 bytes Object IDs.
// The string representation is using URL safe base64 for better space efficiency when
// stored in that form (16 bytes).
//
// UUID is 16 bytes (128 bits), snowflake is 8 bytes (64 bits), xid stands in between
// with 12 bytes with a more compact string representation ready for the web and no
// required configuration or central generation server.
//
// Features:
//
//   - Size: 12 bytes (96 bits), smaller than UUID, larger than snowflake
//   - Base64 URL safe encoded by default (16 bytes storage when transported as printable string)
//   - Non configured, you don't need set a unique machine and/or data center id
//   - K-ordered
//   - Embedded time with 1 second precision
//   - Unicity guaranted for 16,777,216 (24 bits) unique ids per second and per host/process
//
// Best used with xlog's RequestIDHandler (https://godoc.org/github.com/rs/xlog#RequestIDHandler).
//
// References:
//
//   - http://www.slideshare.net/davegardnerisme/unique-id-generation-in-distributed-systems
//   - https://en.wikipedia.org/wiki/Universally_unique_identifier
//   - https://blog.twitter.com/2010/announcing-snowflake
package xid

import (
	"crypto/md5"
	"crypto/rand"
	"encoding/base64"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"os"
	"sync/atomic"
	"time"
)

// Code inspired from mgo/bson ObjectId

// ID represents a unique request id
type ID [rawLen]byte

const (
	encodedLen = 16
	rawLen     = 12
)

// ErrInvalidID is returned when trying to unmarshal an invalid ID
var ErrInvalidID = errors.New("invalid ID")

// objectIDCounter is atomically incremented when generating a new ObjectId
// using NewObjectId() function. It's used as a counter part of an id.
var objectIDCounter uint32

// machineId stores machine id generated once and used in subsequent calls
// to NewObjectId function.
var machineID = readMachineID()

// readMachineId generates machine id and puts it into the machineId global
// variable. If this function fails to get the hostname, it will cause
// a runtime error.
func readMachineID() []byte {
	id := make([]byte, 3)
	hostname, err1 := os.Hostname()
	if err1 != nil {
		// Fallback to rand number if machine id can't be gathered
		_, err2 := io.ReadFull(rand.Reader, id)
		if err2 != nil {
			panic(fmt.Errorf("cannot get hostname: %v; %v", err1, err2))
		}
		return id
	}
	hw := md5.New()
	hw.Write([]byte(hostname))
	copy(id, hw.Sum(nil))
	return id
}

// New generates a globaly unique ID
func New() ID {
	var id ID
	// Timestamp, 4 bytes, big endian
	binary.BigEndian.PutUint32(id[:], uint32(time.Now().Unix()))
	// Machine, first 3 bytes of md5(hostname)
	id[4] = machineID[0]
	id[5] = machineID[1]
	id[6] = machineID[2]
	// Pid, 2 bytes, specs don't specify endianness, but we use big endian.
	pid := os.Getpid()
	id[7] = byte(pid >> 8)
	id[8] = byte(pid)
	// Increment, 3 bytes, big endian
	i := atomic.AddUint32(&objectIDCounter, 1)
	id[9] = byte(i >> 16)
	id[10] = byte(i >> 8)
	id[11] = byte(i)
	return id
}

// String returns a base64 URL safe representation of the id
func (id ID) String() string {
	return base64.URLEncoding.EncodeToString(id[:])
}

// MarshalText implements encoding/text TextMarshaler interface
func (id ID) MarshalText() (text []byte, err error) {
	text = make([]byte, encodedLen)
	base64.URLEncoding.Encode(text, id[:])
	return
}

// UnmarshalText implements encoding/text TextUnmarshaler interface
func (id *ID) UnmarshalText(text []byte) error {
	if len(text) != encodedLen {
		return ErrInvalidID
	}
	b := make([]byte, rawLen)
	_, err := base64.URLEncoding.Decode(b, text)
	for i, c := range b {
		id[i] = c
	}
	return err
}

// Time returns the timestamp part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Time() time.Time {
	// First 4 bytes of ObjectId is 32-bit big-endian seconds from epoch.
	secs := int64(binary.BigEndian.Uint32(id[0:4]))
	return time.Unix(secs, 0)
}

// Machine returns the 3-byte machine id part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Machine() []byte {
	return id[4:7]
}

// Pid returns the process id part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Pid() uint16 {
	return binary.BigEndian.Uint16(id[7:9])
}

// Counter returns the incrementing value part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Counter() int32 {
	b := id[9:12]
	// Counter is stored as big-endian 3-byte value
	return int32(uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2]))
}
<commit_msg>init objectIDCounter to a random value<commit_after>// Package xid is a globally unique id generator suited for web scale
//
// Xid is using Mongo Object ID algorithm to generate globally unique ids:
// https://docs.mongodb.org/manual/reference/object-id/
//
//   - 4-byte value representing the seconds since the Unix epoch,
//   - 3-byte machine identifier,
//   - 2-byte process id, and
//   - 3-byte counter, starting with a random value.
//
// The binary representation of the id is compatible with Mongo 12 bytes Object IDs.
// The string representation is using URL safe base64 for better space efficiency when
// stored in that form (16 bytes).
//
// UUID is 16 bytes (128 bits), snowflake is 8 bytes (64 bits), xid stands in between
// with 12 bytes with a more compact string representation ready for the web and no
// required configuration or central generation server.
//
// Features:
//
//   - Size: 12 bytes (96 bits), smaller than UUID, larger than snowflake
//   - Base64 URL safe encoded by default (16 bytes storage when transported as printable string)
//   - Non configured, you don't need set a unique machine and/or data center id
//   - K-ordered
//   - Embedded time with 1 second precision
//   - Unicity guaranted for 16,777,216 (24 bits) unique ids per second and per host/process
//
// Best used with xlog's RequestIDHandler (https://godoc.org/github.com/rs/xlog#RequestIDHandler).
//
// References:
//
//   - http://www.slideshare.net/davegardnerisme/unique-id-generation-in-distributed-systems
//   - https://en.wikipedia.org/wiki/Universally_unique_identifier
//   - https://blog.twitter.com/2010/announcing-snowflake
package xid

import (
	"crypto/md5"
	"crypto/rand"
	"encoding/base64"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"os"
	"sync/atomic"
	"time"
)

// Code inspired from mgo/bson ObjectId

// ID represents a unique request id
type ID [rawLen]byte

const (
	encodedLen = 16
	rawLen     = 12
)

// ErrInvalidID is returned when trying to unmarshal an invalid ID
var ErrInvalidID = errors.New("invalid ID")

// objectIDCounter is atomically incremented when generating a new ObjectId
// using NewObjectId() function. It's used as a counter part of an id.
var objectIDCounter uint32

// init objectIDCounter to be a random initial value.
func init() {
	b := make([]byte, 3)
	if _, e := io.ReadFull(rand.Reader, b); e != nil {
		panic(e)
	}
	objectIDCounter = uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2])
}

// machineId stores machine id generated once and used in subsequent calls
// to NewObjectId function.
var machineID = readMachineID()

// readMachineId generates machine id and puts it into the machineId global
// variable. If this function fails to get the hostname, it will cause
// a runtime error.
func readMachineID() []byte {
	id := make([]byte, 3)
	hostname, err1 := os.Hostname()
	if err1 != nil {
		// Fallback to rand number if machine id can't be gathered
		_, err2 := io.ReadFull(rand.Reader, id)
		if err2 != nil {
			panic(fmt.Errorf("cannot get hostname: %v; %v", err1, err2))
		}
		return id
	}
	hw := md5.New()
	hw.Write([]byte(hostname))
	copy(id, hw.Sum(nil))
	return id
}

// New generates a globaly unique ID
func New() ID {
	var id ID
	// Timestamp, 4 bytes, big endian
	binary.BigEndian.PutUint32(id[:], uint32(time.Now().Unix()))
	// Machine, first 3 bytes of md5(hostname)
	id[4] = machineID[0]
	id[5] = machineID[1]
	id[6] = machineID[2]
	// Pid, 2 bytes, specs don't specify endianness, but we use big endian.
	pid := os.Getpid()
	id[7] = byte(pid >> 8)
	id[8] = byte(pid)
	// Increment, 3 bytes, big endian
	i := atomic.AddUint32(&objectIDCounter, 1)
	id[9] = byte(i >> 16)
	id[10] = byte(i >> 8)
	id[11] = byte(i)
	return id
}

// String returns a base64 URL safe representation of the id
func (id ID) String() string {
	return base64.URLEncoding.EncodeToString(id[:])
}

// MarshalText implements encoding/text TextMarshaler interface
func (id ID) MarshalText() (text []byte, err error) {
	text = make([]byte, encodedLen)
	base64.URLEncoding.Encode(text, id[:])
	return
}

// UnmarshalText implements encoding/text TextUnmarshaler interface
func (id *ID) UnmarshalText(text []byte) error {
	if len(text) != encodedLen {
		return ErrInvalidID
	}
	b := make([]byte, rawLen)
	_, err := base64.URLEncoding.Decode(b, text)
	for i, c := range b {
		id[i] = c
	}
	return err
}

// Time returns the timestamp part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Time() time.Time {
	// First 4 bytes of ObjectId is 32-bit big-endian seconds from epoch.
	secs := int64(binary.BigEndian.Uint32(id[0:4]))
	return time.Unix(secs, 0)
}

// Machine returns the 3-byte machine id part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Machine() []byte {
	return id[4:7]
}

// Pid returns the process id part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Pid() uint16 {
	return binary.BigEndian.Uint16(id[7:9])
}

// Counter returns the incrementing value part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Counter() int32 {
	b := id[9:12]
	// Counter is stored as big-endian 3-byte value
	return int32(uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2]))
}
<|endoftext|>


<commit_before>package objconv

import (
	"bytes"
	"io"
	"unicode/utf8"
)

// EOL represent end-of-line terminaators.
type EOL string

const (
	// LF is a simple new line sequence.
	LF EOL = "\n"

	// CRLF is a two-byte new line sequence.
	CRLF EOL = "\r\n"
)

// Reader implements the io.Reader interface but reports errors through panics
// instead of returning them.
//
// It's not safe to use the reader concurrently from multiple goroutines.
type Reader struct {
	R io.Reader
	n int
	c [4]byte   // ReadByte and ReadRune buffer
	b [100]byte // ReadLine buffer
}

// NewReader returns a Reader that reads from r.
func NewReader(r io.Reader) *Reader {
	switch v := r.(type) {
	case *Reader:
		return v
	default:
		return &Reader{R: r}
	}
}

// Read reads bytes into b from r, panics if there was an error.
//
// The method returns an error to satisfy the io.Read interface, it will always
// be nil and can be ignored.
func (r *Reader) Read(b []byte) (n int, err error) {
	n, err = r.R.Read(b)

	if n > 0 {
		err = nil
		r.n += n
	} else if err != nil {
		if err == io.EOF && r.n != 0 {
			err = io.ErrUnexpectedEOF
		}
		panic(err)
	}

	return
}

// ReadByte reads a byte from r, panics if there was an error.
//
// The method returns an error to satisfy the io.ByteReader interface, it will
// always be nil and can be ignored.
func (r *Reader) ReadByte() (c byte, err error) {
	r.Read(r.c[:1])
	c = r.c[0]
	return
}

// ReadRune reads a rune from r, panics if there was an error.
//
// The method returns an error to satisfy the io.RuneReader interface, it will
// always be nil and can be ignored.
func (r *Reader) ReadRune() (c rune, n int, err error) {
	b, _ := r.ReadByte()

	if (b & 0x80) == 0 {
		c = rune(b)
		n = 1
		return

	} else if (b & 0xE0) == 0xC0 {
		n = 2

	} else if (b & 0xF0) == 0xE0 {
		n = 3

	} else {
		n = 4
	}

	r.c[0] = b
	r.ReadFull(r.c[1:n])
	c, _ = utf8.DecodeRune(r.c[:n])
	return
}

// ReadLine reads a line ending with eol from r, panics if there was an error.
func (r *Reader) ReadLine(eol EOL) (line []byte) {
	line = r.b[:0]
	end := []byte(eol)

	for !bytes.HasSuffix(line, end) {
		b, _ := r.ReadByte()
		line = append(line, b)
	}

	return line[:len(line)-len(eol)]
}

// ReadFull fills up b with data read from r, panics if there was an error.
func (r *Reader) ReadFull(b []byte) {
	io.ReadFull(r, b)
}

// Writer implements the io.Writer interface.
//
// The writer is used to optimize output operations throughout the objconv
// package and sub-packages. It wraps around a generic io.Writer and capture
// the errors it would return, allowing the encoder to detect whether an
// error occured or not after serializing values. It makes it easier to write
// emitters since they don't need to do error checking in their implementations.
//
// It's not safe to use the writer concurrently from multiple goroutines.
type Writer struct {
	W io.Writer
	e error
	b [64]byte // buffer
}

// NewWriter returns a Writer that reads from r.
func NewWriter(w io.Writer) *Writer {
	switch x := w.(type) {
	case *Writer:
		return x
	default:
		return &Writer{W: w}
	}
}

// Write writes b to w.
func (w *Writer) Write(b []byte) (n int, err error) {
	if err = w.e; err == nil {
		n, err = w.W.Write(b)
		w.e = err
	}
	return
}

// WriteByte writes b to w.
func (w *Writer) WriteByte(b byte) (err error) {
	w.b[0] = b
	_, err = w.Write(w.b[:1])
	return
}

// WriteRune writes r to w.
func (w *Writer) WriteRune(r rune) (n int, err error) {
	return w.Write(w.b[:utf8.EncodeRune(w.b[:], r)])
}

// WriteString writes s to w.
func (w *Writer) WriteString(s string) (n int, err error) {
	// Writes the string by chunks of len(w.b) at a time.
	for len(s) != 0 {
		n1 := len(w.b)
		n2 := len(s)
		n3 := n1
		n4 := 0

		if n3 > n2 {
			n3 = n2
		}

		copy(w.b[:], s[:n3])
		n4, err = w.Write(w.b[:n3])
		n += n4
		s = s[n3:]
	}
	return
}

type runeWriter interface {
	WriteRune(rune) (int, error)
}

type stringWriter interface {
	WriteString(string) (int, error)
}

type counter struct{ n int }

func (c *counter) Write(b []byte) (n int, err error) {
	n = len(b)
	c.n += n
	return
}

func (c *counter) WriteByte(b byte) (err error) {
	c.n++
	return
}

func (c *counter) WriteRune(r rune) (n int, err error) {
	n = utf8.RuneLen(r)
	c.n += n
	return
}

func (c *counter) WriteString(s string) (n int, err error) {
	n = len(s)
	c.n += n
	return
}
<commit_msg>fix typo in comment<commit_after>package objconv

import (
	"bytes"
	"io"
	"unicode/utf8"
)

// EOL represent end-of-line terminators.
type EOL string

const (
	// LF is a simple new line sequence.
	LF EOL = "\n"

	// CRLF is a two-byte new line sequence.
	CRLF EOL = "\r\n"
)

// Reader implements the io.Reader interface but reports errors through panics
// instead of returning them.
//
// It's not safe to use the reader concurrently from multiple goroutines.
type Reader struct {
	R io.Reader
	n int
	c [4]byte   // ReadByte and ReadRune buffer
	b [100]byte // ReadLine buffer
}

// NewReader returns a Reader that reads from r.
func NewReader(r io.Reader) *Reader {
	switch v := r.(type) {
	case *Reader:
		return v
	default:
		return &Reader{R: r}
	}
}

// Read reads bytes into b from r, panics if there was an error.
//
// The method returns an error to satisfy the io.Read interface, it will always
// be nil and can be ignored.
func (r *Reader) Read(b []byte) (n int, err error) {
	n, err = r.R.Read(b)

	if n > 0 {
		err = nil
		r.n += n
	} else if err != nil {
		if err == io.EOF && r.n != 0 {
			err = io.ErrUnexpectedEOF
		}
		panic(err)
	}

	return
}

// ReadByte reads a byte from r, panics if there was an error.
//
// The method returns an error to satisfy the io.ByteReader interface, it will
// always be nil and can be ignored.
func (r *Reader) ReadByte() (c byte, err error) {
	r.Read(r.c[:1])
	c = r.c[0]
	return
}

// ReadRune reads a rune from r, panics if there was an error.
//
// The method returns an error to satisfy the io.RuneReader interface, it will
// always be nil and can be ignored.
func (r *Reader) ReadRune() (c rune, n int, err error) {
	b, _ := r.ReadByte()

	if (b & 0x80) == 0 {
		c = rune(b)
		n = 1
		return

	} else if (b & 0xE0) == 0xC0 {
		n = 2

	} else if (b & 0xF0) == 0xE0 {
		n = 3

	} else {
		n = 4
	}

	r.c[0] = b
	r.ReadFull(r.c[1:n])
	c, _ = utf8.DecodeRune(r.c[:n])
	return
}

// ReadLine reads a line ending with eol from r, panics if there was an error.
func (r *Reader) ReadLine(eol EOL) (line []byte) {
	line = r.b[:0]
	end := []byte(eol)

	for !bytes.HasSuffix(line, end) {
		b, _ := r.ReadByte()
		line = append(line, b)
	}

	return line[:len(line)-len(eol)]
}

// ReadFull fills up b with data read from r, panics if there was an error.
func (r *Reader) ReadFull(b []byte) {
	io.ReadFull(r, b)
}

// Writer implements the io.Writer interface.
//
// The writer is used to optimize output operations throughout the objconv
// package and sub-packages. It wraps around a generic io.Writer and capture
// the errors it would return, allowing the encoder to detect whether an
// error occured or not after serializing values. It makes it easier to write
// emitters since they don't need to do error checking in their implementations.
//
// It's not safe to use the writer concurrently from multiple goroutines.
type Writer struct {
	W io.Writer
	e error
	b [64]byte // buffer
}

// NewWriter returns a Writer that reads from r.
func NewWriter(w io.Writer) *Writer {
	switch x := w.(type) {
	case *Writer:
		return x
	default:
		return &Writer{W: w}
	}
}

// Write writes b to w.
func (w *Writer) Write(b []byte) (n int, err error) {
	if err = w.e; err == nil {
		n, err = w.W.Write(b)
		w.e = err
	}
	return
}

// WriteByte writes b to w.
func (w *Writer) WriteByte(b byte) (err error) {
	w.b[0] = b
	_, err = w.Write(w.b[:1])
	return
}

// WriteRune writes r to w.
func (w *Writer) WriteRune(r rune) (n int, err error) {
	return w.Write(w.b[:utf8.EncodeRune(w.b[:], r)])
}

// WriteString writes s to w.
func (w *Writer) WriteString(s string) (n int, err error) {
	// Writes the string by chunks of len(w.b) at a time.
	for len(s) != 0 {
		n1 := len(w.b)
		n2 := len(s)
		n3 := n1
		n4 := 0

		if n3 > n2 {
			n3 = n2
		}

		copy(w.b[:], s[:n3])
		n4, err = w.Write(w.b[:n3])
		n += n4
		s = s[n3:]
	}
	return
}

type runeWriter interface {
	WriteRune(rune) (int, error)
}

type stringWriter interface {
	WriteString(string) (int, error)
}

type counter struct{ n int }

func (c *counter) Write(b []byte) (n int, err error) {
	n = len(b)
	c.n += n
	return
}

func (c *counter) WriteByte(b byte) (err error) {
	c.n++
	return
}

func (c *counter) WriteRune(r rune) (n int, err error) {
	n = utf8.RuneLen(r)
	c.n += n
	return
}

func (c *counter) WriteString(s string) (n int, err error) {
	n = len(s)
	c.n += n
	return
}
<|endoftext|>


<commit_before>// Copyright 2016 The Periph Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package sysfs

import (
	"fmt"
	"log"
	"testing"

	"periph.io/x/periph/conn/gpio"
)

func ExampleLEDByName() {
	// Commented out due to cycle import.
	//if _, err := host.Init(); err != nil {
	//	log.Fatalf("failed to initialize periph: %v", err)
	//}
	for _, led := range LEDs {
		fmt.Printf("- %s: %s\n", led, led.Function())
	}
	led, err := LEDByName("LED0")
	if err != nil {
		log.Fatalf("failed to find LED: %v", err)
	}
	led.Out(gpio.Low)
}

func TestLEDByName(t *testing.T) {
	if _, err := LEDByName("FOO"); err == nil {
		t.Fail()
	}
}
<commit_msg>sysfs-led: add basic unit test<commit_after>// Copyright 2016 The Periph Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package sysfs

import (
	"fmt"
	"log"
	"testing"

	"periph.io/x/periph/conn/gpio"
)

func ExampleLEDByName() {
	// Commented out due to cycle import.
	//if _, err := host.Init(); err != nil {
	//	log.Fatalf("failed to initialize periph: %v", err)
	//}
	for _, led := range LEDs {
		fmt.Printf("- %s: %s\n", led, led.Function())
	}
	led, err := LEDByName("LED0")
	if err != nil {
		log.Fatalf("failed to find LED: %v", err)
	}
	led.Out(gpio.Low)
}

//

func TestLEDByName(t *testing.T) {
	if _, err := LEDByName("FOO"); err == nil {
		t.Fail()
	}
}

func TestLED(t *testing.T) {
	l := LED{number: 42, name: "Glow", root: "/tmp/led/priv/"}
	if s := l.String(); s != "Glow(42)" {
		t.Fatal(s)
	}
	if s := l.Name(); s != "Glow" {
		t.Fatal(s)
	}
	if n := l.Number(); n != 42 {
		t.Fatal(n)
	}
}

func TestLEDMock(t *testing.T) {
	l := LED{number: 42, name: "Glow", root: "/tmp/led/priv/"}
	if s := l.Function(); s != "LED/Off" {
		t.Fatal(s)
	}
	if err := l.In(gpio.PullNoChange, gpio.NoEdge); err != nil {
		t.Fatal(err)
	}
	if l := l.Read(); l != gpio.Low {
		t.Fatal("need mock")
	}
	if err := l.Out(gpio.High); err == nil {
		t.Fatal("need mock")
	}
}

func TestLED_not_supported(t *testing.T) {
	l := LED{number: 42, name: "Glow", root: "/tmp/led/priv/"}
	if err := l.In(gpio.PullDown, gpio.NoEdge); err == nil {
		t.Fatal("sysfs-led no real In() support")
	}
	if l.WaitForEdge(-1) {
		t.Fatal("not supported")
	}
	if pull := l.Pull(); pull != gpio.PullNoChange {
		t.Fatal(pull)
	}
	if err := l.PWM(0); err == nil {
		t.Fatal("not supported")
	}
}
<|endoftext|>