<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// evolution of: https://github.com/GoogleCloudPlatform/kubernetes/blob/release-0.6/pkg/client/cache/fifo.go
package queue

import (
	"fmt"
	"reflect"
	"sync"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/config"
)

type EventType int

const (
	ADD_EVENT EventType = iota
	UPDATE_EVENT
	DELETE_EVENT
	POP_EVENT
)

type Entry struct {
	Value UniqueCopyable
	Event EventType
}

type Copyable interface {
	// return an independent copy (deep clone) of the current object
	Copy() Copyable
}

type UniqueID interface {
	GetUID() string
}

type UniqueCopyable interface {
	Copyable
	UniqueID
}

func (e *Entry) Copy() Copyable {
	return &Entry{Value: e.Value.Copy().(UniqueCopyable), Event: e.Event}
}

// deliver a message
type pigeon func(msg *Entry)

func dead(msg *Entry) {
	// intentionally blank
}

// HistoricalFIFO receives adds and updates from a Reflector, and puts them in a queue for
// FIFO order processing. If multiple adds/updates of a single item happen while
// an item is in the queue before it has been processed, it will only be
// processed once, and when it is processed, the most recent version will be
// processed. This can't be done with a channel.
type HistoricalFIFO struct {
	lock    sync.RWMutex
	cond    sync.Cond
	items   map[string]*Entry // We depend on the property that items in the queue are in the set.
	queue   []string
	carrier pigeon // may be dead, but never nil
}

// panics if obj doesn't implement UniqueCopyable; otherwise returns the same, typecast object
func checkType(obj interface{}) UniqueCopyable {
	if v, ok := obj.(UniqueCopyable); !ok {
		panic(fmt.Sprintf("Illegal object type, expected UniqueCopyable: %T", obj))
	} else {
		return v
	}
}

// Add inserts an item, and puts it in the queue. The item is only enqueued
// if it doesn't already exist in the set.
func (f *HistoricalFIFO) Add(id string, v interface{}) {
	obj := checkType(v)
	notifications := []*Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entry, exists := f.items[id]; !exists {
		f.queue = append(f.queue, id)
	} else {
		if entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj)
	f.cond.Broadcast()
}

// Update is the same as Add in this implementation.
func (f *HistoricalFIFO) Update(id string, obj interface{}) {
	f.Add(id, obj)
}

// Add the item to the store, but only if there exists a prior entry for
// for the object in the store whose event type matches that given, and then
// only enqueued if it doesn't already exist in the set.
func (f *HistoricalFIFO) Readd(id string, v interface{}, t EventType) {
	obj := checkType(v)
	notifications := []*Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entry, exists := f.items[id]; exists {
		if entry.Event != t {
			return
		} else if entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj)
	f.cond.Broadcast()
}

// Delete removes an item. It doesn't add it to the queue, because
// this implementation assumes the consumer only cares about the objects,
// not the order in which they were created/added.
func (f *HistoricalFIFO) Delete(id string) {
	deleteEvent := (*Entry)(nil)
	defer func() {
		f.carrier(deleteEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	entry, exists := f.items[id]
	if exists {
		//TODO(jdef): set a timer to expunge the history for this object
		//or else, simply do garbage collection every n'th merge(), removing
		//expired DELETE entries
		deleteEvent = &Entry{Value: entry.Value, Event: DELETE_EVENT}
		f.items[id] = deleteEvent
	}
}

// List returns a list of all the items.
func (f *HistoricalFIFO) List() []interface{} {
	f.lock.RLock()
	defer f.lock.RUnlock()

	// TODO(jdef): slightly overallocates b/c of deleted items
	list := make([]interface{}, 0, len(f.queue))

	for _, entry := range f.items {
		if entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			continue
		}
		list = append(list, entry.Value.Copy())
	}
	return list
}

// ContainedIDs returns a util.StringSet containing all IDs of the stored items.
// This is a snapshot of a moment in time, and one should keep in mind that
// other go routines can add or remove items after you call this.
func (c *HistoricalFIFO) ContainedIDs() util.StringSet {
	c.lock.RLock()
	defer c.lock.RUnlock()
	set := util.StringSet{}
	for id, entry := range c.items {
		if entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			continue
		}
		set.Insert(id)
	}
	return set
}

// Get returns the requested item, or sets exists=false.
func (f *HistoricalFIFO) Get(id string) (interface{}, bool) {
	f.lock.RLock()
	defer f.lock.RUnlock()
	entry, exists := f.items[id]
	if exists && !(entry.Event == DELETE_EVENT || entry.Event == POP_EVENT) {
		return entry.Value.Copy(), true
	}
	return nil, false
}

// Pop waits until an item is ready and returns it. If multiple items are
// ready, they are returned in the order in which they were added/updated.
// The item is removed from the queue (and the store) before it is returned,
// so if you don't succesfully process it, you need to add it back with Add().
func (f *HistoricalFIFO) Pop() interface{} {
	popEvent := (*Entry)(nil)
	defer func() {
		f.carrier(popEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	for {
		for len(f.queue) == 0 {
			f.cond.Wait()
		}
		id := f.queue[0]
		f.queue = f.queue[1:]
		entry, ok := f.items[id]
		if !ok || entry.Event == DELETE_EVENT || entry.Event == POP_EVENT {
			// Item may have been deleted subsequently.
			continue
		}
		value := entry.Value
		popEvent = &Entry{Value: value, Event: POP_EVENT}
		f.items[id] = popEvent
		return value.Copy()
	}
}

// Replace will delete the contents of 'f', using instead the given map.
// 'f' takes ownersip of the map, you should not reference the map again
// after calling this function. f's queue is reset, too; upon return, it
// will contain the items in the map, in no particular order.
func (f *HistoricalFIFO) Replace(idToObj map[string]interface{}) {
	notifications := make([]*Entry, 0, len(idToObj))
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	f.queue = f.queue[:0]
	for id, v := range f.items {
		if _, exists := idToObj[id]; !exists && v.Event != DELETE_EVENT {
			// a non-deleted entry in the items list that doesn't show up in the
			// new list: mark it as deleted
			e := &Entry{Value: v.Value, Event: DELETE_EVENT}
			f.items[id] = e
			notifications = append(notifications, e)
		}
	}
	for id, v := range idToObj {
		obj := checkType(v)
		f.queue = append(f.queue, id)
		n := f.merge(id, obj)
		notifications = append(notifications, n...)
	}
	if len(f.queue) > 0 {
		f.cond.Broadcast()
	}
}

// expects that caller has already locked around state
func (f *HistoricalFIFO) merge(id string, obj UniqueCopyable) (notifications []*Entry) {
	entry, exists := f.items[id]
	if !exists {
		e := &Entry{Value: obj.Copy().(UniqueCopyable), Event: ADD_EVENT}
		f.items[id] = e
		notifications = append(notifications, e)
	} else {
		if entry.Event != DELETE_EVENT && entry.Value.GetUID() != obj.GetUID() {
			// hidden DELETE!
			// (1) append a DELETE
			// (2) append an ADD
			// .. and notify listeners in that order
			e1 := &Entry{Value: entry.Value, Event: DELETE_EVENT}
			e2 := &Entry{Value: obj.Copy().(UniqueCopyable), Event: ADD_EVENT}
			f.items[id] = e2
			notifications = append(notifications, e1, e2)
		} else if !reflect.DeepEqual(obj, entry.Value) {
			//TODO(jdef): it would be nice if we could rely on resource versions
			//instead of doing a DeepEqual. Maybe someday we'll be able to.
			e := &Entry{Value: obj.Copy().(UniqueCopyable), Event: UPDATE_EVENT}
			f.items[id] = e
			notifications = append(notifications, e)
		}
	}
	return
}

// NewFIFO returns a Store which can be used to queue up items to
// process. If a non-nil Mux is provided, then modifications to the
// the FIFO are delivered on a channel specific to this fifo.
func NewFIFO(mux *config.Mux) *HistoricalFIFO {
	carrier := dead
	if mux != nil {
		//TODO(jdef): append a UUID to "fifo" here?
		ch := mux.Channel("fifo")
		carrier = func(msg *Entry) {
			if msg != nil {
				ch <- msg.Copy()
			}
		}
	}
	f := &HistoricalFIFO{
		items:   map[string]*Entry{},
		queue:   []string{},
		carrier: carrier,
	}
	f.cond.L = &f.lock
	return f
}
<commit_msg>refactor Entry into an interface; simply the storage map; initial implementation of garbage collection for deleted entries<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// evolution of: https://github.com/GoogleCloudPlatform/kubernetes/blob/release-0.6/pkg/client/cache/fifo.go
package queue

import (
	"fmt"
	"reflect"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
)

type EventType int

const (
	ADD_EVENT EventType = 1 << iota
	UPDATE_EVENT
	DELETE_EVENT
	POP_EVENT
)

type Entry interface {
	Copyable
	Value() UniqueCopyable
	// types is a logically OR'd combination of EventType, e.g. ADD_EVENT|UPDATE_EVENT
	Is(types EventType) bool
}

type entry struct {
	value UniqueCopyable
	event EventType
}

type deletedEntry struct {
	*entry
	expiration time.Time
}

type Copyable interface {
	// return an independent copy (deep clone) of the current object
	Copy() Copyable
}

type UniqueID interface {
	GetUID() string
}

type UniqueCopyable interface {
	Copyable
	UniqueID
}

func (e *entry) Value() UniqueCopyable {
	return e.value
}

func (e *entry) Copy() Copyable {
	if e == nil {
		return nil
	}
	return &entry{e.value.Copy().(UniqueCopyable), e.event}
}

func (e *entry) Is(types EventType) bool {
	return types&e.event != 0
}

func (e *deletedEntry) Copy() Copyable {
	if e == nil {
		return nil
	}
	return &deletedEntry{e.entry.Copy().(*entry), e.expiration}
}

// deliver a message
type pigeon func(msg Entry)

func dead(msg Entry) {
	// intentionally blank
}

// HistoricalFIFO receives adds and updates from a Reflector, and puts them in a queue for
// FIFO order processing. If multiple adds/updates of a single item happen while
// an item is in the queue before it has been processed, it will only be
// processed once, and when it is processed, the most recent version will be
// processed. This can't be done with a channel.
type HistoricalFIFO struct {
	lock      sync.RWMutex
	cond      sync.Cond
	items     map[string]Entry // We depend on the property that items in the queue are in the set.
	queue     []string
	carrier   pigeon // may be dead, but never nil
	gcc       int
	lingerTTL time.Duration
}

// panics if obj doesn't implement UniqueCopyable; otherwise returns the same, typecast object
func checkType(obj interface{}) UniqueCopyable {
	if v, ok := obj.(UniqueCopyable); !ok {
		panic(fmt.Sprintf("Illegal object type, expected UniqueCopyable: %T", obj))
	} else {
		return v
	}
}

// Add inserts an item, and puts it in the queue. The item is only enqueued
// if it doesn't already exist in the set.
func (f *HistoricalFIFO) Add(id string, v interface{}) {
	obj := checkType(v)
	notifications := []Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entry, exists := f.items[id]; !exists {
		f.queue = append(f.queue, id)
	} else {
		if entry.Is(DELETE_EVENT | POP_EVENT) {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj)
	f.cond.Broadcast()
}

// Update is the same as Add in this implementation.
func (f *HistoricalFIFO) Update(id string, obj interface{}) {
	f.Add(id, obj)
}

// Add the item to the store, but only if there exists a prior entry for
// for the object in the store whose event type matches that given, and then
// only enqueued if it doesn't already exist in the set.
func (f *HistoricalFIFO) Readd(id string, v interface{}, t EventType) {
	obj := checkType(v)
	notifications := []Entry(nil)
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	if entry, exists := f.items[id]; exists {
		if !entry.Is(t) {
			return
		} else if entry.Is(DELETE_EVENT | POP_EVENT) {
			f.queue = append(f.queue, id)
		}
	}
	notifications = f.merge(id, obj)
	f.cond.Broadcast()
}

// Delete removes an item. It doesn't add it to the queue, because
// this implementation assumes the consumer only cares about the objects,
// not the order in which they were created/added.
func (f *HistoricalFIFO) Delete(id string) {
	deleteEvent := (Entry)(nil)
	defer func() {
		f.carrier(deleteEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	item, exists := f.items[id]
	if exists && !item.Is(DELETE_EVENT) {
		e := item.(*entry)
		e.event = DELETE_EVENT
		deleteEvent = &deletedEntry{e, time.Now().Add(f.lingerTTL)}
		f.items[id] = deleteEvent
	}
}

// List returns a list of all the items.
func (f *HistoricalFIFO) List() []interface{} {
	f.lock.RLock()
	defer f.lock.RUnlock()

	// TODO(jdef): slightly overallocates b/c of deleted items
	list := make([]interface{}, 0, len(f.queue))

	for _, entry := range f.items {
		if entry.Is(DELETE_EVENT | POP_EVENT) {
			continue
		}
		list = append(list, entry.Value().Copy())
	}
	return list
}

// ContainedIDs returns a util.StringSet containing all IDs of the stored items.
// This is a snapshot of a moment in time, and one should keep in mind that
// other go routines can add or remove items after you call this.
func (c *HistoricalFIFO) ContainedIDs() util.StringSet {
	c.lock.RLock()
	defer c.lock.RUnlock()
	set := util.StringSet{}
	for id, entry := range c.items {
		if entry.Is(DELETE_EVENT | POP_EVENT) {
			continue
		}
		set.Insert(id)
	}
	return set
}

// Get returns the requested item, or sets exists=false.
func (f *HistoricalFIFO) Get(id string) (interface{}, bool) {
	f.lock.RLock()
	defer f.lock.RUnlock()
	entry, exists := f.items[id]
	if exists && !entry.Is(DELETE_EVENT|POP_EVENT) {
		return entry.Value().Copy(), true
	}
	return nil, false
}

// Pop waits until an item is ready and returns it. If multiple items are
// ready, they are returned in the order in which they were added/updated.
// The item is removed from the queue (and the store) before it is returned,
// so if you don't succesfully process it, you need to add it back with Add().
func (f *HistoricalFIFO) Pop() interface{} {
	popEvent := (Entry)(nil)
	defer func() {
		f.carrier(popEvent)
	}()

	f.lock.Lock()
	defer f.lock.Unlock()
	for {
		for len(f.queue) == 0 {
			f.cond.Wait()
		}
		id := f.queue[0]
		f.queue = f.queue[1:]
		item, ok := f.items[id]
		if !ok || item.Is(DELETE_EVENT|POP_EVENT) {
			// Item may have been deleted subsequently.
			continue
		}
		value := item.Value()
		popEvent = &entry{value, POP_EVENT}
		f.items[id] = popEvent
		return value.Copy()
	}
}

// Replace will delete the contents of 'f', using instead the given map.
// 'f' takes ownersip of the map, you should not reference the map again
// after calling this function. f's queue is reset, too; upon return, it
// will contain the items in the map, in no particular order.
func (f *HistoricalFIFO) Replace(idToObj map[string]interface{}) {
	notifications := make([]Entry, 0, len(idToObj))
	defer func() {
		for _, e := range notifications {
			f.carrier(e)
		}
	}()

	f.lock.Lock()
	defer f.lock.Unlock()

	f.queue = f.queue[:0]
	now := time.Now()
	for id, v := range f.items {
		if _, exists := idToObj[id]; !exists && !v.Is(DELETE_EVENT) {
			// a non-deleted entry in the items list that doesn't show up in the
			// new list: mark it as deleted
			ent := v.(*entry)
			ent.event = DELETE_EVENT
			e := &deletedEntry{ent, now.Add(f.lingerTTL)}
			f.items[id] = e
			notifications = append(notifications, e)
		}
	}
	for id, v := range idToObj {
		obj := checkType(v)
		f.queue = append(f.queue, id)
		n := f.merge(id, obj)
		notifications = append(notifications, n...)
	}
	if len(f.queue) > 0 {
		f.cond.Broadcast()
	}
}

// garbage collect DELETEd items whose TTL has expired; the IDs of such items are removed
// from the queue. This impl assumes that caller has acquired state lock.
func (f *HistoricalFIFO) gc() {
	now := time.Now()
	deleted := make(map[string]struct{})
	for id, v := range f.items {
		if v.Is(DELETE_EVENT) {
			ent := v.(*deletedEntry)
			if ent.expiration.Before(now) {
				delete(f.items, id)
				deleted[id] = struct{}{}
			}
		}
	}
	// remove deleted items from the queue, will likely (slightly) overallocate here
	queue := make([]string, 0, len(f.queue))
	for _, id := range f.queue {
		if _, exists := deleted[id]; !exists {
			queue = append(queue, id)
		}
	}
	f.queue = queue
}

// Assumes that the caller has acquired the state lock.
func (f *HistoricalFIFO) merge(id string, obj UniqueCopyable) (notifications []Entry) {
	item, exists := f.items[id]
	now := time.Now()
	if !exists {
		e := &entry{obj.Copy().(UniqueCopyable), ADD_EVENT}
		f.items[id] = e
		notifications = append(notifications, e)
	} else {
		if !item.Is(DELETE_EVENT) && item.Value().GetUID() != obj.GetUID() {
			// hidden DELETE!
			// (1) append a DELETE
			// (2) append an ADD
			// .. and notify listeners in that order
			ent := item.(*entry)
			ent.event = DELETE_EVENT
			e1 := &deletedEntry{ent, now.Add(f.lingerTTL)}
			e2 := &entry{obj.Copy().(UniqueCopyable), ADD_EVENT}
			f.items[id] = e2
			notifications = append(notifications, e1, e2)
		} else if !reflect.DeepEqual(obj, item.Value()) {
			//TODO(jdef): it would be nice if we could rely on resource versions
			//instead of doing a DeepEqual. Maybe someday we'll be able to.
			e := &entry{obj.Copy().(UniqueCopyable), UPDATE_EVENT}
			f.items[id] = e
			notifications = append(notifications, e)
		}
	}
	// check for garbage collection
	f.gcc++
	if f.gcc%256 == 0 { //TODO(jdef): extract constant
		f.gcc = 0
		f.gc()
	}
	return
}

// NewFIFO returns a Store which can be used to queue up items to
// process. If a non-nil Mux is provided, then modifications to the
// the FIFO are delivered on a channel specific to this fifo.
func NewFIFO(ch chan<- Entry) *HistoricalFIFO {
	carrier := dead
	if ch != nil {
		carrier = func(msg Entry) {
			if msg != nil {
				ch <- msg.Copy().(Entry)
			}
		}
	}
	f := &HistoricalFIFO{
		items:     map[string]Entry{},
		queue:     []string{},
		carrier:   carrier,
		lingerTTL: 5 * time.Minute, // TODO(jdef): extract constant
	}
	f.cond.L = &f.lock
	return f
}
<|endoftext|>


<commit_before>// Code generated by "stringer -type=Type"; DO NOT EDIT

package gel

import "fmt"

const _Type_name = "TextualElementAttributeNodeListAttributeList"

var _Type_index = [...]uint8{0, 7, 14, 23, 31, 44}

func (i Type) String() string {
	i -= 1
	if i < 0 || i >= Type(len(_Type_index)-1) {
		return fmt.Sprintf("Type(%d)", i+1)
	}
	return _Type_name[_Type_index[i]:_Type_index[i+1]]
}
<commit_msg>Merging<commit_after><|endoftext|>


<commit_before>package linereader

import (
	"bytes"
	"io"
	"log"
	"sync"
	"time"
)

type LineReader struct {
	Reader     io.Reader
	Buffer     *bytes.Buffer
	CacheBytes []byte
	LeftBytes  []byte
	Mutex      sync.Mutex
}

func NewLineReader(reader io.Reader) *LineReader {
	return &LineReader{
		Reader:     reader,
		Buffer:     bytes.NewBufferString(""),
		CacheBytes: make([]byte, 0, 500),
		LeftBytes:  make([]byte, 0, 50),
	}
}

func (lineReader *LineReader) Line() string {
	line, err := lineReader.Buffer.ReadBytes('\n')
	if err != nil {
		if err == io.EOF {
			lineReader.Mutex.Lock()
			lineReader.LeftBytes = append(lineReader.LeftBytes, line...)
			log.Println(line)
			lineReader.Mutex.Unlock()
		}
		log.Println("line string error:", err.Error())
		time.Sleep(time.Second * 1)
		return lineReader.Line()
	}

	return string(line)
}

func (lineReader *LineReader) Reading() {
	p := make([]byte, 1000)
	for {
		n, err := lineReader.Reader.Read(p)
		if err != nil {
			log.Println("line reading error%v", err)
		}

		if n > 0 {
			lineReader.WriteBuffer(p[:n])
		}

		// if there is not enough content to read
		if n < 300 {
			time.Sleep(time.Second * 1)
		}
	}
}

func (lineReader *LineReader) WriteBuffer(b []byte) {
	n := len(b)

	// Enlarge our cache bytes to match what we were handed, and copy
	// our current buffer over
	cbLen := len(lineReader.CacheBytes)
	if cap(lineReader.CacheBytes) < cap(b) {
		newBuf := make([]byte, cbLen, cap(b))
		copy(newBuf, lineReader.CacheBytes)
		lineReader.CacheBytes = newBuf
	}

	//TODO the Cache will lose by unexpect method log lose < 2 item
	// Write the current buffer if the new data won't fit
	if cbLen+n > cap(lineReader.CacheBytes) {
		lineReader.Mutex.Lock()
		lineReader.Buffer.Write(lineReader.LeftBytes)
		lineReader.LeftBytes = make([]byte, 0, 50)
		lineReader.Buffer.Write(lineReader.CacheBytes[:cbLen])
		lineReader.CacheBytes = make([]byte, 0, cap(b))
		cbLen = 0
		lineReader.Mutex.Unlock()
	}

	// Copy the new data into our saveBuffer
	lineReader.CacheBytes = lineReader.CacheBytes[:cbLen+n]
	copy(lineReader.CacheBytes[cbLen:], b)
}
<commit_msg>remove line log<commit_after>package linereader

import (
	"bytes"
	"io"
	"log"
	"sync"
	"time"
)

type LineReader struct {
	Reader     io.Reader
	Buffer     *bytes.Buffer
	CacheBytes []byte
	LeftBytes  []byte
	Mutex      sync.Mutex
}

func NewLineReader(reader io.Reader) *LineReader {
	return &LineReader{
		Reader:     reader,
		Buffer:     bytes.NewBufferString(""),
		CacheBytes: make([]byte, 0, 500),
		LeftBytes:  make([]byte, 0, 50),
	}
}

func (lineReader *LineReader) Line() string {
	line, err := lineReader.Buffer.ReadBytes('\n')
	if err != nil {
		if err == io.EOF {
			lineReader.Mutex.Lock()
			lineReader.LeftBytes = append(lineReader.LeftBytes, line...)
			lineReader.Mutex.Unlock()
		}
		log.Println("line string error:", err.Error())
		time.Sleep(time.Second * 1)
		return lineReader.Line()
	}

	return string(line)
}

func (lineReader *LineReader) Reading() {
	p := make([]byte, 1000)
	for {
		n, err := lineReader.Reader.Read(p)
		if err != nil {
			log.Println("line reading error%v", err)
		}

		if n > 0 {
			lineReader.WriteBuffer(p[:n])
		}

		// if there is not enough content to read
		if n < 300 {
			time.Sleep(time.Second * 1)
		}
	}
}

func (lineReader *LineReader) WriteBuffer(b []byte) {
	n := len(b)

	// Enlarge our cache bytes to match what we were handed, and copy
	// our current buffer over
	cbLen := len(lineReader.CacheBytes)
	if cap(lineReader.CacheBytes) < cap(b) {
		newBuf := make([]byte, cbLen, cap(b))
		copy(newBuf, lineReader.CacheBytes)
		lineReader.CacheBytes = newBuf
	}

	//TODO the Cache will lose by unexpect method log lose < 2 item
	// Write the current buffer if the new data won't fit
	if cbLen+n > cap(lineReader.CacheBytes) {
		lineReader.Mutex.Lock()
		lineReader.Buffer.Write(lineReader.LeftBytes)
		lineReader.LeftBytes = make([]byte, 0, 50)
		lineReader.Buffer.Write(lineReader.CacheBytes[:cbLen])
		lineReader.CacheBytes = make([]byte, 0, cap(b))
		cbLen = 0
		lineReader.Mutex.Unlock()
	}

	// Copy the new data into our saveBuffer
	lineReader.CacheBytes = lineReader.CacheBytes[:cbLen+n]
	copy(lineReader.CacheBytes[cbLen:], b)
}
<|endoftext|>


<commit_before>package utils

import (
	"math"
	"regexp"

	"github.com/lfq7413/tomato/livequery/t"
)

// QueryHash ...
func QueryHash(query t.M) string {
	return ""
}

// MatchesQuery ...
func MatchesQuery(object, query t.M) bool {

	if className, ok := query["className"]; ok {
		if object["className"].(string) != className {
			return false
		}
		return MatchesQuery(object, query["where"].(map[string]interface{}))
	}

	for field, constraints := range query {
		if matchesKeyConstraints(object, field, constraints) == false {
			return false
		}
	}

	return true
}

func matchesKeyConstraints(object t.M, key string, constraints interface{}) bool {
	if constraints == nil {
		return false
	}
	if key == "$or" {
		if querys, ok := constraints.([]interface{}); ok {
			for _, query := range querys {
				if q, ok := query.(map[string]interface{}); ok {
					if MatchesQuery(object, q) {
						return true
					}
				}
			}
			return false
		}
		return false
	}

	// 不支持 relatedTo
	if key == "$relatedTo" {
		return false
	}

	// 只支持 key == "$or" 时，constraints 为数组的情况
	if _, ok := constraints.([]interface{}); ok {
		return false
	}

	var constraint t.M
	if v, ok := constraints.(map[string]interface{}); ok {
		constraint = v
	} else {
		if objects, ok := object[key].([]interface{}); ok {
			for _, o := range objects {
				if equalObject(o, constraints) {
					return true
				}
			}
			return false
		}
		return equalObject(object[key], constraints)
	}

	objectType := constraint["__type"].(string)
	if objectType != "" {
		if objectType == "Pointer" {
			o := object[key].(map[string]interface{})
			if o == nil {
				return false
			}
			class1 := constraint["className"]
			class2 := o["className"]
			id1 := constraint["objectId"]
			id2 := o["objectId"]
			return class1 == class2 && id1 == id2
		}

		if objects, ok := object[key].([]interface{}); ok {
			for _, object := range objects {
				if equalObject(object, constraint) {
					return true
				}
			}
			return false
		}
		return equalObject(object[key], constraint)
	}

	for condition, compareTo := range constraint {
		switch condition {
		case "$lt", "$lte", "$gt", "$gte":
			if compareNumber(object[key], compareTo, condition) == false {
				return false
			}
		case "$ne":
			if equalObject(object[key], compareTo) {
				return false
			}
		case "$in":
			if inSlice(compareTo, object[key]) == false {
				return false
			}
		case "$nin":
			if inSlice(compareTo, object[key]) {
				return false
			}
		case "$all":
			if compareToObjects, ok := compareTo.([]interface{}); ok {
				for _, compareToObject := range compareToObjects {
					if inSlice(object[key], compareToObject) == false {
						return false
					}
				}
			} else {
				return false
			}
		case "$exists":
			var propertyExists bool
			if _, ok := object[key]; ok {
				propertyExists = true
			} else {
				propertyExists = false
			}
			var existenceIsRequired bool
			if v, ok := constraint["$exists"].(bool); ok {
				existenceIsRequired = v
			} else {
				break
			}
			if (!propertyExists && existenceIsRequired) || (propertyExists && !existenceIsRequired) {
				return false
			}
		case "$regex":
			if compareRegexp(compareTo, object[key]) == false {
				return false
			}
		case "$nearSphere":
			if compareGeoPoint(compareTo, object[key], constraint["$maxDistance"]) == false {
				return false
			}
		case "$within":
			if compareBox(compareTo, object[key]) == false {
				return false
			}
		case "$options":
		case "$maxDistance":
		case "$select":
			return false
		case "$dontSelect":
			return false
		default:
			return false
		}
	}

	return true
}

func compareBox(compareTo, point interface{}) bool {
	southWest := map[string]float64{}
	northEast := map[string]float64{}
	geoPoint := map[string]float64{}

	box := []interface{}{}
	if within, ok := compareTo.(map[string]interface{}); ok {
		if b, ok := within["$box"].([]interface{}); ok {
			box = b
		} else {
			return false
		}
	} else {
		return false
	}

	if len(box) != 2 {
		return false
	}

	if p, ok := box[0].(map[string]interface{}); ok {
		southWest["longitude"] = p["longitude"].(float64)
		southWest["latitude"] = p["latitude"].(float64)
	} else {
		return false
	}

	if p, ok := box[1].(map[string]interface{}); ok {
		northEast["longitude"] = p["longitude"].(float64)
		northEast["latitude"] = p["latitude"].(float64)
	} else {
		return false
	}

	if southWest["latitude"] > northEast["latitude"] ||
		southWest["longitude"] > northEast["longitude"] {
		return false
	}

	if p, ok := point.(map[string]interface{}); ok {
		geoPoint["longitude"] = p["longitude"].(float64)
		geoPoint["latitude"] = p["latitude"].(float64)
	} else {
		return false
	}

	return geoPoint["latitude"] > southWest["latitude"] &&
		geoPoint["latitude"] < northEast["latitude"] &&
		geoPoint["longitude"] > southWest["longitude"] &&
		geoPoint["longitude"] < northEast["longitude"]
}

func compareGeoPoint(p1, p2, maxDistance interface{}) bool {
	if v1, ok := p1.(map[string]interface{}); ok {
		if v2, ok := p2.(map[string]interface{}); ok {
			if x1, ok := v1["longitude"].(float64); ok {
				if y1, ok := v1["latitude"].(float64); ok {
					if x2, ok := v2["longitude"].(float64); ok {
						if y2, ok := v2["latitude"].(float64); ok {
							d := distance(x1, y1, x2, y2)
							if maxDistance == nil {
								return true
							}
							if max, ok := maxDistance.(float64); ok {
								return d <= max
							}
						}
					}
				}
			}
		}
	}
	return false
}

// distance 计算两点之间的弧度值，两点经纬度分别为 (x1,y1), (x2,y2)
func distance(x1, y1, x2, y2 float64) float64 {
	x1 = x1 * math.Pi / 180
	y1 = y1 * math.Pi / 180
	x2 = x2 * math.Pi / 180
	y2 = y2 * math.Pi / 180
	// d=R*arcos[cos(Y1)*cos(Y2)*cos(X1-X2)+sin(Y1)*sin(Y2)]
	return math.Acos(math.Cos(y1)*math.Cos(y2)*math.Cos(x1-x2) + math.Sin(y1)*math.Sin(y2))
}

// compareRegexp 比较 object 是否符合正则表达式 exp
func compareRegexp(exp, object interface{}) bool {
	if c, ok := exp.(string); ok {
		if o, ok := object.(string); ok {
			matched, err := regexp.MatchString(c, o)
			if err != nil {
				return false
			}
			return matched
		}
		return false
	}
	return false
}

// inSlice 判断 slice 中是否存在指定的对象
// s 必须为 []interface{} 类型
// o 要查找的对象
func inSlice(s, o interface{}) bool {
	if objects, ok := s.([]interface{}); ok {
		for _, object := range objects {
			if equalObject(object, o) {
				return true
			}
		}
		return false
	}
	return false
}

// compareNumber i1, i2 均为数字类型时才比较大小
func compareNumber(i1, i2 interface{}, op string) bool {
	if v1, ok := i1.(float64); ok {
		if v2, ok := i2.(float64); ok {
			switch op {
			case "$lt":
				return v1 < v2
			case "$lte":
				return v1 <= v2
			case "$gt":
				return v1 > v2
			case "$gte":
				return v1 >= v2
			default:
				return false
			}
		}
		return false
	}
	return false
}

// equalObject 仅比较基础类型：string float64 bool slice map
func equalObject(i1, i2 interface{}) bool {
	if v1, ok := i1.(string); ok {
		if v2, ok := i2.(string); ok {
			return v1 == v2
		}
		return false
	}

	if v1, ok := i1.(float64); ok {
		if v2, ok := i2.(float64); ok {
			return v1 == v2
		}
		return false
	}

	if v1, ok := i1.(bool); ok {
		if v2, ok := i2.(bool); ok {
			return v1 == v2
		}
		return false
	}

	if v1, ok := i1.([]interface{}); ok {
		if v2, ok := i2.([]interface{}); ok {
			if len(v1) != len(v2) {
				return false
			}
			for i := 0; i < len(v1); i++ {
				if equalObject(v1[i], v2[i]) == false {
					return false
				}
			}
			return true
		}
		return false
	}

	if v1, ok := i1.(map[string]interface{}); ok {
		if v2, ok := i2.(map[string]interface{}); ok {
			if len(v1) != len(v2) {
				return false
			}
			for k := range v1 {
				if equalObject(v1[k], v2[k]) == false {
					return false
				}
			}
			return true
		}
		return false
	}

	return false
}
<commit_msg>完成 QueryHash 、 flattenOrQueries<commit_after>package utils

import (
	"fmt"
	"math"
	"regexp"
	"sort"
	"strings"

	"github.com/lfq7413/tomato/livequery/t"
)

// QueryHash 计算 query 的 hash
func QueryHash(query t.M) string {
	where := query["where"].(map[string]interface{})
	if where == nil {
		where = map[string]interface{}{}
	}

	list := flattenOrQueries(where)
	columns := []string{}
	values := []interface{}{}
	if len(list) > 0 {
		uniqueColumns := map[string]bool{}
		for _, subQuery := range list {
			keys := []string{}
			for k := range subQuery {
				keys = append(keys, k)
			}
			sort.Sort(sort.StringSlice(keys))
			for _, k := range keys {
				values = append(values, subQuery[k])
				uniqueColumns[k] = true
			}
		}
		for k := range uniqueColumns {
			columns = append(columns, k)
		}
		sort.Sort(sort.StringSlice(columns))
	} else {
		for k := range where {
			columns = append(columns, k)
		}
		sort.Sort(sort.StringSlice(columns))
		for _, k := range columns {
			values = append(values, where[k])
		}
	}

	sections := []string{strings.Join(columns, ","), fmt.Sprint(values)}

	return query["className"].(string) + ":" + strings.Join(sections, "|")
}

// flattenOrQueries 处理 where 中的 $or 字段
func flattenOrQueries(where t.M) []map[string]interface{} {
	ors := where["$or"]
	if ors == nil {
		return nil
	}
	if v, ok := ors.([]interface{}); ok {
		accum := []map[string]interface{}{}
		for _, sub := range v {
			if query, ok := sub.(map[string]interface{}); ok {
				accum = append(accum, query)
			}
		}
		return accum
	}
	return nil
}

// MatchesQuery ...
func MatchesQuery(object, query t.M) bool {

	if className, ok := query["className"]; ok {
		if object["className"].(string) != className {
			return false
		}
		return MatchesQuery(object, query["where"].(map[string]interface{}))
	}

	for field, constraints := range query {
		if matchesKeyConstraints(object, field, constraints) == false {
			return false
		}
	}

	return true
}

func matchesKeyConstraints(object t.M, key string, constraints interface{}) bool {
	if constraints == nil {
		return false
	}
	if key == "$or" {
		if querys, ok := constraints.([]interface{}); ok {
			for _, query := range querys {
				if q, ok := query.(map[string]interface{}); ok {
					if MatchesQuery(object, q) {
						return true
					}
				}
			}
			return false
		}
		return false
	}

	// 不支持 relatedTo
	if key == "$relatedTo" {
		return false
	}

	// 只支持 key == "$or" 时，constraints 为数组的情况
	if _, ok := constraints.([]interface{}); ok {
		return false
	}

	var constraint t.M
	if v, ok := constraints.(map[string]interface{}); ok {
		constraint = v
	} else {
		if objects, ok := object[key].([]interface{}); ok {
			for _, o := range objects {
				if equalObject(o, constraints) {
					return true
				}
			}
			return false
		}
		return equalObject(object[key], constraints)
	}

	objectType := constraint["__type"].(string)
	if objectType != "" {
		if objectType == "Pointer" {
			o := object[key].(map[string]interface{})
			if o == nil {
				return false
			}
			class1 := constraint["className"]
			class2 := o["className"]
			id1 := constraint["objectId"]
			id2 := o["objectId"]
			return class1 == class2 && id1 == id2
		}

		if objects, ok := object[key].([]interface{}); ok {
			for _, object := range objects {
				if equalObject(object, constraint) {
					return true
				}
			}
			return false
		}
		return equalObject(object[key], constraint)
	}

	for condition, compareTo := range constraint {
		switch condition {
		case "$lt", "$lte", "$gt", "$gte":
			if compareNumber(object[key], compareTo, condition) == false {
				return false
			}
		case "$ne":
			if equalObject(object[key], compareTo) {
				return false
			}
		case "$in":
			if inSlice(compareTo, object[key]) == false {
				return false
			}
		case "$nin":
			if inSlice(compareTo, object[key]) {
				return false
			}
		case "$all":
			if compareToObjects, ok := compareTo.([]interface{}); ok {
				for _, compareToObject := range compareToObjects {
					if inSlice(object[key], compareToObject) == false {
						return false
					}
				}
			} else {
				return false
			}
		case "$exists":
			var propertyExists bool
			if _, ok := object[key]; ok {
				propertyExists = true
			} else {
				propertyExists = false
			}
			var existenceIsRequired bool
			if v, ok := constraint["$exists"].(bool); ok {
				existenceIsRequired = v
			} else {
				break
			}
			if (!propertyExists && existenceIsRequired) || (propertyExists && !existenceIsRequired) {
				return false
			}
		case "$regex":
			if compareRegexp(compareTo, object[key]) == false {
				return false
			}
		case "$nearSphere":
			if compareGeoPoint(compareTo, object[key], constraint["$maxDistance"]) == false {
				return false
			}
		case "$within":
			if compareBox(compareTo, object[key]) == false {
				return false
			}
		case "$options":
		case "$maxDistance":
		case "$select":
			return false
		case "$dontSelect":
			return false
		default:
			return false
		}
	}

	return true
}

func compareBox(compareTo, point interface{}) bool {
	southWest := map[string]float64{}
	northEast := map[string]float64{}
	geoPoint := map[string]float64{}

	box := []interface{}{}
	if within, ok := compareTo.(map[string]interface{}); ok {
		if b, ok := within["$box"].([]interface{}); ok {
			box = b
		} else {
			return false
		}
	} else {
		return false
	}

	if len(box) != 2 {
		return false
	}

	if p, ok := box[0].(map[string]interface{}); ok {
		southWest["longitude"] = p["longitude"].(float64)
		southWest["latitude"] = p["latitude"].(float64)
	} else {
		return false
	}

	if p, ok := box[1].(map[string]interface{}); ok {
		northEast["longitude"] = p["longitude"].(float64)
		northEast["latitude"] = p["latitude"].(float64)
	} else {
		return false
	}

	if southWest["latitude"] > northEast["latitude"] ||
		southWest["longitude"] > northEast["longitude"] {
		return false
	}

	if p, ok := point.(map[string]interface{}); ok {
		geoPoint["longitude"] = p["longitude"].(float64)
		geoPoint["latitude"] = p["latitude"].(float64)
	} else {
		return false
	}

	return geoPoint["latitude"] > southWest["latitude"] &&
		geoPoint["latitude"] < northEast["latitude"] &&
		geoPoint["longitude"] > southWest["longitude"] &&
		geoPoint["longitude"] < northEast["longitude"]
}

func compareGeoPoint(p1, p2, maxDistance interface{}) bool {
	if v1, ok := p1.(map[string]interface{}); ok {
		if v2, ok := p2.(map[string]interface{}); ok {
			if x1, ok := v1["longitude"].(float64); ok {
				if y1, ok := v1["latitude"].(float64); ok {
					if x2, ok := v2["longitude"].(float64); ok {
						if y2, ok := v2["latitude"].(float64); ok {
							d := distance(x1, y1, x2, y2)
							if maxDistance == nil {
								return true
							}
							if max, ok := maxDistance.(float64); ok {
								return d <= max
							}
						}
					}
				}
			}
		}
	}
	return false
}

// distance 计算两点之间的弧度值，两点经纬度分别为 (x1,y1), (x2,y2)
func distance(x1, y1, x2, y2 float64) float64 {
	x1 = x1 * math.Pi / 180
	y1 = y1 * math.Pi / 180
	x2 = x2 * math.Pi / 180
	y2 = y2 * math.Pi / 180
	// d=R*arcos[cos(Y1)*cos(Y2)*cos(X1-X2)+sin(Y1)*sin(Y2)]
	return math.Acos(math.Cos(y1)*math.Cos(y2)*math.Cos(x1-x2) + math.Sin(y1)*math.Sin(y2))
}

// compareRegexp 比较 object 是否符合正则表达式 exp
func compareRegexp(exp, object interface{}) bool {
	if c, ok := exp.(string); ok {
		if o, ok := object.(string); ok {
			matched, err := regexp.MatchString(c, o)
			if err != nil {
				return false
			}
			return matched
		}
		return false
	}
	return false
}

// inSlice 判断 slice 中是否存在指定的对象
// s 必须为 []interface{} 类型
// o 要查找的对象
func inSlice(s, o interface{}) bool {
	if objects, ok := s.([]interface{}); ok {
		for _, object := range objects {
			if equalObject(object, o) {
				return true
			}
		}
		return false
	}
	return false
}

// compareNumber i1, i2 均为数字类型时才比较大小
func compareNumber(i1, i2 interface{}, op string) bool {
	if v1, ok := i1.(float64); ok {
		if v2, ok := i2.(float64); ok {
			switch op {
			case "$lt":
				return v1 < v2
			case "$lte":
				return v1 <= v2
			case "$gt":
				return v1 > v2
			case "$gte":
				return v1 >= v2
			default:
				return false
			}
		}
		return false
	}
	return false
}

// equalObject 仅比较基础类型：string float64 bool slice map
func equalObject(i1, i2 interface{}) bool {
	if v1, ok := i1.(string); ok {
		if v2, ok := i2.(string); ok {
			return v1 == v2
		}
		return false
	}

	if v1, ok := i1.(float64); ok {
		if v2, ok := i2.(float64); ok {
			return v1 == v2
		}
		return false
	}

	if v1, ok := i1.(bool); ok {
		if v2, ok := i2.(bool); ok {
			return v1 == v2
		}
		return false
	}

	if v1, ok := i1.([]interface{}); ok {
		if v2, ok := i2.([]interface{}); ok {
			if len(v1) != len(v2) {
				return false
			}
			for i := 0; i < len(v1); i++ {
				if equalObject(v1[i], v2[i]) == false {
					return false
				}
			}
			return true
		}
		return false
	}

	if v1, ok := i1.(map[string]interface{}); ok {
		if v2, ok := i2.(map[string]interface{}); ok {
			if len(v1) != len(v2) {
				return false
			}
			for k := range v1 {
				if equalObject(v1[k], v2[k]) == false {
					return false
				}
			}
			return true
		}
		return false
	}

	return false
}
<|endoftext|>


<commit_before>package d1

const Model = ""
<commit_msg>nikon/d1: Add Model string<commit_after>package d1

const Model = "NIKON D1"
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package race_test

import (
	"testing"
)

func TestRaceMapRW(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		_ = m[1]
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestRaceMapRW2(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		_, _ = m[1]
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestRaceMapRWArray(t *testing.T) {
	// Check instrumentation of unaddressable arrays (issue 4578).
	m := make(map[int][2]int)
	ch := make(chan bool, 1)
	go func() {
		_ = m[1][1]
		ch <- true
	}()
	m[2] = [2]int{1, 2}
	<-ch
}

func TestNoRaceMapRR(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		_, _ = m[1]
		ch <- true
	}()
	_ = m[1]
	<-ch
}

func TestRaceMapRange(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		for range m {
		}
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestRaceMapRange2(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		for range m {
		}
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestNoRaceMapRangeRange(t *testing.T) {
	m := make(map[int]int)
	// now the map is not empty and range triggers an event
	// should work without this (as in other tests)
	// so it is suspicious if this test passes and others don't
	m[0] = 0
	ch := make(chan bool, 1)
	go func() {
		for range m {
		}
		ch <- true
	}()
	for range m {
	}
	<-ch
}

func TestRaceMapLen(t *testing.T) {
	m := make(map[string]bool)
	ch := make(chan bool, 1)
	go func() {
		_ = len(m)
		ch <- true
	}()
	m[""] = true
	<-ch
}

func TestRaceMapDelete(t *testing.T) {
	m := make(map[string]bool)
	ch := make(chan bool, 1)
	go func() {
		delete(m, "")
		ch <- true
	}()
	m[""] = true
	<-ch
}

func TestRaceMapLenDelete(t *testing.T) {
	m := make(map[string]bool)
	ch := make(chan bool, 1)
	go func() {
		delete(m, "a")
		ch <- true
	}()
	_ = len(m)
	<-ch
}

func TestRaceMapVariable(t *testing.T) {
	ch := make(chan bool, 1)
	m := make(map[int]int)
	go func() {
		m = make(map[int]int)
		ch <- true
	}()
	m = make(map[int]int)
	<-ch
}

func TestRaceMapVariable2(t *testing.T) {
	ch := make(chan bool, 1)
	m := make(map[int]int)
	go func() {
		m[1] = 1
		ch <- true
	}()
	m = make(map[int]int)
	<-ch
}

func TestRaceMapVariable3(t *testing.T) {
	ch := make(chan bool, 1)
	m := make(map[int]int)
	go func() {
		_ = m[1]
		ch <- true
	}()
	m = make(map[int]int)
	<-ch
}

type Big struct {
	x [17]int32
}

func TestRaceMapLookupPartKey(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	_ = m[*k]
	<-ch
}

func TestRaceMapLookupPartKey2(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	_, _ = m[*k]
	<-ch
}
func TestRaceMapDeletePartKey(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	delete(m, *k)
	<-ch
}

func TestRaceMapInsertPartKey(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	m[*k] = true
	<-ch
}

func TestRaceMapInsertPartVal(t *testing.T) {
	v := &Big{}
	m := make(map[int]Big)
	ch := make(chan bool, 1)
	go func() {
		v.x[8] = 1
		ch <- true
	}()
	m[1] = *v
	<-ch
}

// Test for issue 7561.
func TestRaceMapAssignMultipleReturn(t *testing.T) {
	connect := func() (int, error) { return 42, nil }
	conns := make(map[int][]int)
	conns[1] = []int{0}
	ch := make(chan bool, 1)
	var err error
	go func() {
		conns[1][0], err = connect()
		ch <- true
	}()
	x := conns[1][0]
	_ = x
	<-ch
}
<commit_msg>runtime/race: add tests for maps with big keys/vals With the recent GC changes large objects are handled somewhat differently.<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package race_test

import (
	"testing"
)

func TestRaceMapRW(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		_ = m[1]
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestRaceMapRW2(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		_, _ = m[1]
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestRaceMapRWArray(t *testing.T) {
	// Check instrumentation of unaddressable arrays (issue 4578).
	m := make(map[int][2]int)
	ch := make(chan bool, 1)
	go func() {
		_ = m[1][1]
		ch <- true
	}()
	m[2] = [2]int{1, 2}
	<-ch
}

func TestNoRaceMapRR(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		_, _ = m[1]
		ch <- true
	}()
	_ = m[1]
	<-ch
}

func TestRaceMapRange(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		for range m {
		}
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestRaceMapRange2(t *testing.T) {
	m := make(map[int]int)
	ch := make(chan bool, 1)
	go func() {
		for range m {
		}
		ch <- true
	}()
	m[1] = 1
	<-ch
}

func TestNoRaceMapRangeRange(t *testing.T) {
	m := make(map[int]int)
	// now the map is not empty and range triggers an event
	// should work without this (as in other tests)
	// so it is suspicious if this test passes and others don't
	m[0] = 0
	ch := make(chan bool, 1)
	go func() {
		for range m {
		}
		ch <- true
	}()
	for range m {
	}
	<-ch
}

func TestRaceMapLen(t *testing.T) {
	m := make(map[string]bool)
	ch := make(chan bool, 1)
	go func() {
		_ = len(m)
		ch <- true
	}()
	m[""] = true
	<-ch
}

func TestRaceMapDelete(t *testing.T) {
	m := make(map[string]bool)
	ch := make(chan bool, 1)
	go func() {
		delete(m, "")
		ch <- true
	}()
	m[""] = true
	<-ch
}

func TestRaceMapLenDelete(t *testing.T) {
	m := make(map[string]bool)
	ch := make(chan bool, 1)
	go func() {
		delete(m, "a")
		ch <- true
	}()
	_ = len(m)
	<-ch
}

func TestRaceMapVariable(t *testing.T) {
	ch := make(chan bool, 1)
	m := make(map[int]int)
	go func() {
		m = make(map[int]int)
		ch <- true
	}()
	m = make(map[int]int)
	<-ch
}

func TestRaceMapVariable2(t *testing.T) {
	ch := make(chan bool, 1)
	m := make(map[int]int)
	go func() {
		m[1] = 1
		ch <- true
	}()
	m = make(map[int]int)
	<-ch
}

func TestRaceMapVariable3(t *testing.T) {
	ch := make(chan bool, 1)
	m := make(map[int]int)
	go func() {
		_ = m[1]
		ch <- true
	}()
	m = make(map[int]int)
	<-ch
}

type Big struct {
	x [17]int32
}

func TestRaceMapLookupPartKey(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	_ = m[*k]
	<-ch
}

func TestRaceMapLookupPartKey2(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	_, _ = m[*k]
	<-ch
}
func TestRaceMapDeletePartKey(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	delete(m, *k)
	<-ch
}

func TestRaceMapInsertPartKey(t *testing.T) {
	k := &Big{}
	m := make(map[Big]bool)
	ch := make(chan bool, 1)
	go func() {
		k.x[8] = 1
		ch <- true
	}()
	m[*k] = true
	<-ch
}

func TestRaceMapInsertPartVal(t *testing.T) {
	v := &Big{}
	m := make(map[int]Big)
	ch := make(chan bool, 1)
	go func() {
		v.x[8] = 1
		ch <- true
	}()
	m[1] = *v
	<-ch
}

// Test for issue 7561.
func TestRaceMapAssignMultipleReturn(t *testing.T) {
	connect := func() (int, error) { return 42, nil }
	conns := make(map[int][]int)
	conns[1] = []int{0}
	ch := make(chan bool, 1)
	var err error
	go func() {
		conns[1][0], err = connect()
		ch <- true
	}()
	x := conns[1][0]
	_ = x
	<-ch
}

// BigKey and BigVal must be larger than 256 bytes,
// so that compiler sets KindGCProg for them.
type BigKey [1000]*int

type BigVal struct {
	x int
	y [1000]*int
}

func TestRaceMapBigKeyAccess1(t *testing.T) {
	m := make(map[BigKey]int)
	var k BigKey
	ch := make(chan bool, 1)
	go func() {
		_ = m[k]
		ch <- true
	}()
	k[30] = new(int)
	<-ch
}

func TestRaceMapBigKeyAccess2(t *testing.T) {
	m := make(map[BigKey]int)
	var k BigKey
	ch := make(chan bool, 1)
	go func() {
		_, _ = m[k]
		ch <- true
	}()
	k[30] = new(int)
	<-ch
}

func TestRaceMapBigKeyInsert(t *testing.T) {
	m := make(map[BigKey]int)
	var k BigKey
	ch := make(chan bool, 1)
	go func() {
		m[k] = 1
		ch <- true
	}()
	k[30] = new(int)
	<-ch
}

func TestRaceMapBigKeyDelete(t *testing.T) {
	m := make(map[BigKey]int)
	var k BigKey
	ch := make(chan bool, 1)
	go func() {
		delete(m, k)
		ch <- true
	}()
	k[30] = new(int)
	<-ch
}

func TestRaceMapBigValInsert(t *testing.T) {
	m := make(map[int]BigVal)
	var v BigVal
	ch := make(chan bool, 1)
	go func() {
		m[1] = v
		ch <- true
	}()
	v.y[30] = new(int)
	<-ch
}

func TestRaceMapBigValAccess1(t *testing.T) {
	m := make(map[int]BigVal)
	var v BigVal
	ch := make(chan bool, 1)
	go func() {
		v = m[1]
		ch <- true
	}()
	v.y[30] = new(int)
	<-ch
}

func TestRaceMapBigValAccess2(t *testing.T) {
	m := make(map[int]BigVal)
	var v BigVal
	ch := make(chan bool, 1)
	go func() {
		v, _ = m[1]
		ch <- true
	}()
	v.y[30] = new(int)
	<-ch
}
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2014, Google Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *     * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

/*
Package transport defines and implements message oriented communication channel
to complete various transactions (e.g., an RPC).
*/
package transport // import "google.golang.org/grpc/transport"

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"net"
	"sync"

	"golang.org/x/net/context"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
)

// recvMsg represents the received msg from the transport. All transport
// protocol specific info has been removed.
type recvMsg struct {
	data []byte
	// nil: received some data
	// io.EOF: stream is completed. data is nil.
	// other non-nil error: transport failure. data is nil.
	err error
}

func (recvMsg) isItem() bool {
	return true
}

// All items in an out of a recvBuffer should be the same type.
type item interface {
	isItem() bool
}

// recvBuffer is an unbounded channel of item.
type recvBuffer struct {
	c       chan item
	mu      sync.Mutex
	backlog []item
}

func newRecvBuffer() *recvBuffer {
	b := &recvBuffer{
		c: make(chan item, 1),
	}
	return b
}

func (b *recvBuffer) put(r item) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.backlog = append(b.backlog, r)
	select {
	case b.c <- b.backlog[0]:
		b.backlog = b.backlog[1:]
	default:
	}
}

func (b *recvBuffer) load() {
	b.mu.Lock()
	defer b.mu.Unlock()
	if len(b.backlog) > 0 {
		select {
		case b.c <- b.backlog[0]:
			b.backlog = b.backlog[1:]
		default:
		}
	}
}

// get returns the channel that receives an item in the buffer.
//
// Upon receipt of an item, the caller should call load to send another
// item onto the channel if there is any.
func (b *recvBuffer) get() <-chan item {
	return b.c
}

// recvBufferReader implements io.Reader interface to read the data from
// recvBuffer.
type recvBufferReader struct {
	ctx  context.Context
	recv *recvBuffer
	last *bytes.Reader // Stores the remaining data in the previous calls.
	err  error
}

// Read reads the next len(p) bytes from last. If last is drained, it tries to
// read additional data from recv. It blocks if there no additional data available
// in recv. If Read returns any non-nil error, it will continue to return that error.
func (r *recvBufferReader) Read(p []byte) (n int, err error) {
	if r.err != nil {
		return 0, r.err
	}
	defer func() { r.err = err }()
	if r.last != nil && r.last.Len() > 0 {
		// Read remaining data left in last call.
		return r.last.Read(p)
	}
	select {
	case <-r.ctx.Done():
		return 0, ContextErr(r.ctx.Err())
	case i := <-r.recv.get():
		r.recv.load()
		m := i.(*recvMsg)
		if m.err != nil {
			return 0, m.err
		}
		r.last = bytes.NewReader(m.data)
		return r.last.Read(p)
	}
}

type streamState uint8

const (
	streamActive    streamState = iota
	streamWriteDone             // EndStream sent
	streamReadDone              // EndStream received
	streamDone                  // sendDone and recvDone or RSTStreamFrame is sent or received.
)

// Stream represents an RPC in the transport layer.
type Stream struct {
	id uint32
	// nil for client side Stream.
	st ServerTransport
	// ctx is the associated context of the stream.
	ctx    context.Context
	cancel context.CancelFunc
	// method records the associated RPC method of the stream.
	method string
	buf    *recvBuffer
	dec    io.Reader

	// Inbound quota for flow control
	recvQuota int
	// The handler to control the window update procedure for both this
	// particular stream and the associated transport.
	windowHandler func(int)

	sendQuotaPool *quotaPool
	// Close headerChan to indicate the end of reception of header metadata.
	headerChan chan struct{}
	// header caches the received header metadata.
	header metadata.MD
	// The key-value map of trailer metadata.
	trailer metadata.MD

	mu sync.RWMutex
	// headerOK becomes true from the first header is about to send.
	headerOk bool
	state    streamState
	// true iff headerChan is closed. Used to avoid closing headerChan
	// multiple times.
	headerDone bool
	// the status received from the server.
	statusCode codes.Code
	statusDesc string
}

// Header acquires the key-value pairs of header metadata once it
// is available. It blocks until i) the metadata is ready or ii) there is no
// header metadata or iii) the stream is cancelled/expired.
func (s *Stream) Header() (metadata.MD, error) {
	select {
	case <-s.ctx.Done():
		return nil, ContextErr(s.ctx.Err())
	case <-s.headerChan:
		return s.header.Copy(), nil
	}
}

// Trailer returns the cached trailer metedata. Note that if it is not called
// after the entire stream is done, it could return an empty MD. Client
// side only.
func (s *Stream) Trailer() metadata.MD {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.trailer.Copy()
}

// ServerTransport returns the underlying ServerTransport for the stream.
// The client side stream always returns nil.
func (s *Stream) ServerTransport() ServerTransport {
	return s.st
}

// Context returns the context of the stream.
func (s *Stream) Context() context.Context {
	return s.ctx
}

// Method returns the method for the stream.
func (s *Stream) Method() string {
	return s.method
}

// StatusCode returns statusCode received from the server.
func (s *Stream) StatusCode() codes.Code {
	return s.statusCode
}

// StatusDesc returns statusDesc received from the server.
func (s *Stream) StatusDesc() string {
	return s.statusDesc
}

// ErrIllegalTrailerSet indicates that the trailer has already been set or it
// is too late to do so.
var ErrIllegalTrailerSet = errors.New("grpc/transport: trailer has been set")

// SetTrailer sets the trailer metadata which will be sent with the RPC status
// by the server. This can only be called at most once. Server side only.
func (s *Stream) SetTrailer(md metadata.MD) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.trailer != nil {
		return ErrIllegalTrailerSet
	}
	s.trailer = md.Copy()
	return nil
}

func (s *Stream) write(m recvMsg) {
	s.buf.put(&m)
}

// Read reads all the data available for this Stream from the transport and
// passes them into the decoder, which converts them into a gRPC message stream.
// The error is io.EOF when the stream is done or another non-nil error if
// the stream broke.
func (s *Stream) Read(p []byte) (n int, err error) {
	n, err = s.dec.Read(p)
	if err != nil {
		return
	}
	s.windowHandler(n)
	return
}

type key int

// The key to save transport.Stream in the context.
const streamKey = key(0)

// newContextWithStream creates a new context from ctx and attaches stream
// to it.
func newContextWithStream(ctx context.Context, stream *Stream) context.Context {
	return context.WithValue(ctx, streamKey, stream)
}

// StreamFromContext returns the stream saved in ctx.
func StreamFromContext(ctx context.Context) (s *Stream, ok bool) {
	s, ok = ctx.Value(streamKey).(*Stream)
	return
}

// state of transport
type transportState int

const (
	reachable transportState = iota
	unreachable
	closing
)

// NewServerTransport creates a ServerTransport with conn or non-nil error
// if it fails.
func NewServerTransport(protocol string, conn net.Conn, maxStreams uint32) (ServerTransport, error) {
	return newHTTP2Server(conn, maxStreams)
}

// NewClientTransport establishes the transport with the required protocol
// and returns it to the caller.
func NewClientTransport(protocol, target string, authOpts []credentials.Credentials) (ClientTransport, error) {
	return newHTTP2Client(target, authOpts)
}

// Options provides additional hints and information for message
// transmission.
type Options struct {
	// Indicate whether it is the last piece for this stream.
	Last bool
	// The hint to transport impl whether the data could be buffered for
	// batching write. Transport impl can feel free to ignore it.
	Delay bool
}

// CallHdr carries the metadata from client to server.
type CallHdr struct {
	Host   string // peer host
	Method string // the operation to perform on the specified host
}

// ClientTransport is the common interface for all gRPC client side transport
// implementations.
type ClientTransport interface {
	// Close tears down this transport. Once it returns, the transport
	// should not be accessed any more. The caller must make sure this
	// is called only once.
	Close() error

	// Write sends the data for the given stream. A nil stream indicates
	// the write is to be performed on the transport as a whole.
	Write(s *Stream, data []byte, opts *Options) error

	// NewStream creates a Stream for an RPC.
	NewStream(ctx context.Context, callHdr *CallHdr) (*Stream, error)

	// CloseStream clears the footprint of a stream when the stream is
	// not needed any more. The err indicates the error incurred when
	// CloseStream is called. Must be called when a stream is finished
	// unless the associated transport is closing.
	CloseStream(stream *Stream, err error)

	// Error returns a channel that is closed when some I/O error
	// happens. Typically the caller should have a goroutine to monitor
	// this in order to take action (e.g., close the current transport
	// and create a new one) in error case. It should not return nil
	// once the transport is initiated.
	Error() <-chan struct{}
}

// ServerTransport is the common interface for all gRPC server side transport
// implementations.
type ServerTransport interface {
	// WriteStatus sends the status of a stream to the client.
	WriteStatus(s *Stream, statusCode codes.Code, statusDesc string) error
	// Write sends the data for the given stream.
	Write(s *Stream, data []byte, opts *Options) error
	// WriteHeader sends the header metedata for the given stream.
	WriteHeader(s *Stream, md metadata.MD) error
	// HandleStreams receives incoming streams using the given handler.
	HandleStreams(func(*Stream))
	// Close tears down the transport. Once it is called, the transport
	// should not be accessed any more. All the pending streams and their
	// handlers will be terminated asynchronously.
	Close() error
}

// StreamErrorf creates an StreamError with the specified error code and description.
func StreamErrorf(c codes.Code, format string, a ...interface{}) StreamError {
	return StreamError{
		Code: c,
		Desc: fmt.Sprintf(format, a...),
	}
}

// ConnectionErrorf creates an ConnectionError with the specified error description.
func ConnectionErrorf(format string, a ...interface{}) ConnectionError {
	return ConnectionError{
		Desc: fmt.Sprintf(format, a...),
	}
}

// ConnectionError is an error that results in the termination of the
// entire connection and the retry of all the active streams.
type ConnectionError struct {
	Desc string
}

func (e ConnectionError) Error() string {
	return fmt.Sprintf("connection error: desc = %q", e.Desc)
}

// Define some common ConnectionErrors.
var ErrConnClosing = ConnectionError{Desc: "transport is closing"}

// StreamError is an error that only affects one stream within a connection.
type StreamError struct {
	Code codes.Code
	Desc string
}

func (e StreamError) Error() string {
	return fmt.Sprintf("stream error: code = %d desc = %q", e.Code, e.Desc)
}

// ContextErr converts the error from context package into a StreamError.
func ContextErr(err error) StreamError {
	switch err {
	case context.DeadlineExceeded:
		return StreamErrorf(codes.DeadlineExceeded, "%v", err)
	case context.Canceled:
		return StreamErrorf(codes.Canceled, "%v", err)
	}
	panic(fmt.Sprintf("Unexpected error from context packet: %v", err))
}

// wait blocks until it can receive from ctx.Done, closing, or proceed.
// If it receives from ctx.Done, it returns 0, the StreamError for ctx.Err.
// If it receives from closing, it returns 0, ErrConnClosing.
// If it receives from proceed, it returns the received integer, nil.
func wait(ctx context.Context, closing <-chan struct{}, proceed <-chan int) (int, error) {
	select {
	case <-ctx.Done():
		return 0, ContextErr(ctx.Err())
	case <-closing:
		return 0, ErrConnClosing
	case i := <-proceed:
		return i, nil
	}
}
<commit_msg>fix a comment<commit_after>/*
 *
 * Copyright 2014, Google Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above
 * copyright notice, this list of conditions and the following disclaimer
 * in the documentation and/or other materials provided with the
 * distribution.
 *     * Neither the name of Google Inc. nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

/*
Package transport defines and implements message oriented communication channel
to complete various transactions (e.g., an RPC).
*/
package transport // import "google.golang.org/grpc/transport"

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"net"
	"sync"

	"golang.org/x/net/context"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
)

// recvMsg represents the received msg from the transport. All transport
// protocol specific info has been removed.
type recvMsg struct {
	data []byte
	// nil: received some data
	// io.EOF: stream is completed. data is nil.
	// other non-nil error: transport failure. data is nil.
	err error
}

func (recvMsg) isItem() bool {
	return true
}

// All items in an out of a recvBuffer should be the same type.
type item interface {
	isItem() bool
}

// recvBuffer is an unbounded channel of item.
type recvBuffer struct {
	c       chan item
	mu      sync.Mutex
	backlog []item
}

func newRecvBuffer() *recvBuffer {
	b := &recvBuffer{
		c: make(chan item, 1),
	}
	return b
}

func (b *recvBuffer) put(r item) {
	b.mu.Lock()
	defer b.mu.Unlock()
	b.backlog = append(b.backlog, r)
	select {
	case b.c <- b.backlog[0]:
		b.backlog = b.backlog[1:]
	default:
	}
}

func (b *recvBuffer) load() {
	b.mu.Lock()
	defer b.mu.Unlock()
	if len(b.backlog) > 0 {
		select {
		case b.c <- b.backlog[0]:
			b.backlog = b.backlog[1:]
		default:
		}
	}
}

// get returns the channel that receives an item in the buffer.
//
// Upon receipt of an item, the caller should call load to send another
// item onto the channel if there is any.
func (b *recvBuffer) get() <-chan item {
	return b.c
}

// recvBufferReader implements io.Reader interface to read the data from
// recvBuffer.
type recvBufferReader struct {
	ctx  context.Context
	recv *recvBuffer
	last *bytes.Reader // Stores the remaining data in the previous calls.
	err  error
}

// Read reads the next len(p) bytes from last. If last is drained, it tries to
// read additional data from recv. It blocks if there no additional data available
// in recv. If Read returns any non-nil error, it will continue to return that error.
func (r *recvBufferReader) Read(p []byte) (n int, err error) {
	if r.err != nil {
		return 0, r.err
	}
	defer func() { r.err = err }()
	if r.last != nil && r.last.Len() > 0 {
		// Read remaining data left in last call.
		return r.last.Read(p)
	}
	select {
	case <-r.ctx.Done():
		return 0, ContextErr(r.ctx.Err())
	case i := <-r.recv.get():
		r.recv.load()
		m := i.(*recvMsg)
		if m.err != nil {
			return 0, m.err
		}
		r.last = bytes.NewReader(m.data)
		return r.last.Read(p)
	}
}

type streamState uint8

const (
	streamActive    streamState = iota
	streamWriteDone             // EndStream sent
	streamReadDone              // EndStream received
	streamDone                  // sendDone and recvDone or RSTStreamFrame is sent or received.
)

// Stream represents an RPC in the transport layer.
type Stream struct {
	id uint32
	// nil for client side Stream.
	st ServerTransport
	// ctx is the associated context of the stream.
	ctx    context.Context
	cancel context.CancelFunc
	// method records the associated RPC method of the stream.
	method string
	buf    *recvBuffer
	dec    io.Reader

	// Inbound quota for flow control
	recvQuota int
	// The handler to control the window update procedure for both this
	// particular stream and the associated transport.
	windowHandler func(int)

	sendQuotaPool *quotaPool
	// Close headerChan to indicate the end of reception of header metadata.
	headerChan chan struct{}
	// header caches the received header metadata.
	header metadata.MD
	// The key-value map of trailer metadata.
	trailer metadata.MD

	mu sync.RWMutex
	// headerOK becomes true from the first header is about to send.
	headerOk bool
	state    streamState
	// true iff headerChan is closed. Used to avoid closing headerChan
	// multiple times.
	headerDone bool
	// the status received from the server.
	statusCode codes.Code
	statusDesc string
}

// Header acquires the key-value pairs of header metadata once it
// is available. It blocks until i) the metadata is ready or ii) there is no
// header metadata or iii) the stream is cancelled/expired.
func (s *Stream) Header() (metadata.MD, error) {
	select {
	case <-s.ctx.Done():
		return nil, ContextErr(s.ctx.Err())
	case <-s.headerChan:
		return s.header.Copy(), nil
	}
}

// Trailer returns the cached trailer metedata. Note that if it is not called
// after the entire stream is done, it could return an empty MD. Client
// side only.
func (s *Stream) Trailer() metadata.MD {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.trailer.Copy()
}

// ServerTransport returns the underlying ServerTransport for the stream.
// The client side stream always returns nil.
func (s *Stream) ServerTransport() ServerTransport {
	return s.st
}

// Context returns the context of the stream.
func (s *Stream) Context() context.Context {
	return s.ctx
}

// Method returns the method for the stream.
func (s *Stream) Method() string {
	return s.method
}

// StatusCode returns statusCode received from the server.
func (s *Stream) StatusCode() codes.Code {
	return s.statusCode
}

// StatusDesc returns statusDesc received from the server.
func (s *Stream) StatusDesc() string {
	return s.statusDesc
}

// ErrIllegalTrailerSet indicates that the trailer has already been set or it
// is too late to do so.
var ErrIllegalTrailerSet = errors.New("grpc/transport: trailer has been set")

// SetTrailer sets the trailer metadata which will be sent with the RPC status
// by the server. This can only be called at most once. Server side only.
func (s *Stream) SetTrailer(md metadata.MD) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.trailer != nil {
		return ErrIllegalTrailerSet
	}
	s.trailer = md.Copy()
	return nil
}

func (s *Stream) write(m recvMsg) {
	s.buf.put(&m)
}

// Read reads all the data available for this Stream from the transport and
// passes them into the decoder, which converts them into a gRPC message stream.
// The error is io.EOF when the stream is done or another non-nil error if
// the stream broke.
func (s *Stream) Read(p []byte) (n int, err error) {
	n, err = s.dec.Read(p)
	if err != nil {
		return
	}
	s.windowHandler(n)
	return
}

type key int

// The key to save transport.Stream in the context.
const streamKey = key(0)

// newContextWithStream creates a new context from ctx and attaches stream
// to it.
func newContextWithStream(ctx context.Context, stream *Stream) context.Context {
	return context.WithValue(ctx, streamKey, stream)
}

// StreamFromContext returns the stream saved in ctx.
func StreamFromContext(ctx context.Context) (s *Stream, ok bool) {
	s, ok = ctx.Value(streamKey).(*Stream)
	return
}

// state of transport
type transportState int

const (
	reachable transportState = iota
	unreachable
	closing
)

// NewServerTransport creates a ServerTransport with conn or non-nil error
// if it fails.
func NewServerTransport(protocol string, conn net.Conn, maxStreams uint32) (ServerTransport, error) {
	return newHTTP2Server(conn, maxStreams)
}

// NewClientTransport establishes the transport with the required protocol
// and returns it to the caller.
func NewClientTransport(protocol, target string, authOpts []credentials.Credentials) (ClientTransport, error) {
	return newHTTP2Client(target, authOpts)
}

// Options provides additional hints and information for message
// transmission.
type Options struct {
	// Indicate whether it is the last piece for this stream.
	Last bool
	// The hint to transport impl whether the data could be buffered for
	// batching write. Transport impl can feel free to ignore it.
	Delay bool
}

// CallHdr carries the information of a particular RPC.
type CallHdr struct {
	Host   string // peer host
	Method string // the operation to perform on the specified host
}

// ClientTransport is the common interface for all gRPC client side transport
// implementations.
type ClientTransport interface {
	// Close tears down this transport. Once it returns, the transport
	// should not be accessed any more. The caller must make sure this
	// is called only once.
	Close() error

	// Write sends the data for the given stream. A nil stream indicates
	// the write is to be performed on the transport as a whole.
	Write(s *Stream, data []byte, opts *Options) error

	// NewStream creates a Stream for an RPC.
	NewStream(ctx context.Context, callHdr *CallHdr) (*Stream, error)

	// CloseStream clears the footprint of a stream when the stream is
	// not needed any more. The err indicates the error incurred when
	// CloseStream is called. Must be called when a stream is finished
	// unless the associated transport is closing.
	CloseStream(stream *Stream, err error)

	// Error returns a channel that is closed when some I/O error
	// happens. Typically the caller should have a goroutine to monitor
	// this in order to take action (e.g., close the current transport
	// and create a new one) in error case. It should not return nil
	// once the transport is initiated.
	Error() <-chan struct{}
}

// ServerTransport is the common interface for all gRPC server side transport
// implementations.
type ServerTransport interface {
	// WriteStatus sends the status of a stream to the client.
	WriteStatus(s *Stream, statusCode codes.Code, statusDesc string) error
	// Write sends the data for the given stream.
	Write(s *Stream, data []byte, opts *Options) error
	// WriteHeader sends the header metedata for the given stream.
	WriteHeader(s *Stream, md metadata.MD) error
	// HandleStreams receives incoming streams using the given handler.
	HandleStreams(func(*Stream))
	// Close tears down the transport. Once it is called, the transport
	// should not be accessed any more. All the pending streams and their
	// handlers will be terminated asynchronously.
	Close() error
}

// StreamErrorf creates an StreamError with the specified error code and description.
func StreamErrorf(c codes.Code, format string, a ...interface{}) StreamError {
	return StreamError{
		Code: c,
		Desc: fmt.Sprintf(format, a...),
	}
}

// ConnectionErrorf creates an ConnectionError with the specified error description.
func ConnectionErrorf(format string, a ...interface{}) ConnectionError {
	return ConnectionError{
		Desc: fmt.Sprintf(format, a...),
	}
}

// ConnectionError is an error that results in the termination of the
// entire connection and the retry of all the active streams.
type ConnectionError struct {
	Desc string
}

func (e ConnectionError) Error() string {
	return fmt.Sprintf("connection error: desc = %q", e.Desc)
}

// Define some common ConnectionErrors.
var ErrConnClosing = ConnectionError{Desc: "transport is closing"}

// StreamError is an error that only affects one stream within a connection.
type StreamError struct {
	Code codes.Code
	Desc string
}

func (e StreamError) Error() string {
	return fmt.Sprintf("stream error: code = %d desc = %q", e.Code, e.Desc)
}

// ContextErr converts the error from context package into a StreamError.
func ContextErr(err error) StreamError {
	switch err {
	case context.DeadlineExceeded:
		return StreamErrorf(codes.DeadlineExceeded, "%v", err)
	case context.Canceled:
		return StreamErrorf(codes.Canceled, "%v", err)
	}
	panic(fmt.Sprintf("Unexpected error from context packet: %v", err))
}

// wait blocks until it can receive from ctx.Done, closing, or proceed.
// If it receives from ctx.Done, it returns 0, the StreamError for ctx.Err.
// If it receives from closing, it returns 0, ErrConnClosing.
// If it receives from proceed, it returns the received integer, nil.
func wait(ctx context.Context, closing <-chan struct{}, proceed <-chan int) (int, error) {
	select {
	case <-ctx.Done():
		return 0, ContextErr(ctx.Err())
	case <-closing:
		return 0, ErrConnClosing
	case i := <-proceed:
		return i, nil
	}
}
<|endoftext|>


<commit_before>package cache

import (
	"context"
	"errors"
	"sync/atomic"
	"time"

	"github.com/go-redis/cache/v7/internal"
	"github.com/go-redis/cache/v7/internal/lrucache"
	"github.com/go-redis/cache/v7/internal/singleflight"

	"github.com/go-redis/redis/v7"
)

var ErrCacheMiss = errors.New("cache: key is missing")
var errRedisLocalCacheNil = errors.New("cache: both Redis and LocalCache are nil")

func SetLogger(logger internal.Logger) {
	internal.Log = logger
}

type rediser interface {
	Set(key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	Get(key string) *redis.StringCmd
	Del(keys ...string) *redis.IntCmd
}

type Item struct {
	Ctx context.Context

	Key    string
	Object interface{}

	// Func returns object to be cached.
	Func func() (interface{}, error)

	// Expiration is the cache expiration time.
	// Default expiration is 1 hour.
	Expiration time.Duration
}

func (item *Item) object() (interface{}, error) {
	if item.Object != nil {
		return item.Object, nil
	}
	if item.Func != nil {
		return item.Func()
	}
	return nil, nil
}

func (item *Item) exp() time.Duration {
	if item.Expiration < 0 {
		return 0
	}
	if item.Expiration < time.Second {
		return time.Hour
	}
	return item.Expiration
}

type Codec struct {
	Redis rediser

	localCache *lrucache.Cache

	Marshal   func(interface{}) ([]byte, error)
	Unmarshal func([]byte, interface{}) error

	group singleflight.Group

	hits        uint64
	misses      uint64
	localHits   uint64
	localMisses uint64
}

// UseLocalCache causes Codec to cache items in local LRU cache.
func (cd *Codec) UseLocalCache(maxLen int, expiration time.Duration) {
	cd.localCache = lrucache.New(maxLen, expiration)
}

// Set caches the item.
func (cd *Codec) Set(item *Item) error {
	_, err := cd.setItem(item)
	return err
}

func (cd *Codec) setItem(item *Item) ([]byte, error) {
	object, err := item.object()
	if err != nil {
		return nil, err
	}

	b, err := cd.Marshal(object)
	if err != nil {
		internal.Log.Printf("cache: Marshal key=%q failed: %s", item.Key, err)
		return nil, err
	}

	if cd.localCache != nil {
		cd.localCache.Set(item.Key, b)
	}

	if cd.Redis == nil {
		if cd.localCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return b, nil
	}

	err = cd.Redis.Set(item.Key, b, item.exp()).Err()
	if err != nil {
		internal.Log.Printf("cache: Set key=%q failed: %s", item.Key, err)
	}
	return b, err
}

// Exists reports whether object for the given key exists.
func (cd *Codec) Exists(key string) bool {
	return cd.Get(key, nil) == nil
}

func (cd *Codec) ExistsContext(ctx context.Context, key string) bool {
	return cd.GetContext(ctx, key, nil) == nil
}

// Get gets the object for the given key.
func (cd *Codec) Get(key string, object interface{}) error {
	return cd.get(context.Background(), key, object, false)
}

func (cd *Codec) GetContext(ctx context.Context, key string, object interface{}) error {
	return cd.get(ctx, key, object, false)
}

func (cd *Codec) get(
	ctx context.Context, key string, object interface{}, onlyLocalCache bool,
) error {
	b, err := cd.getBytes(key, onlyLocalCache)
	if err != nil {
		return err
	}

	if object == nil || len(b) == 0 {
		return nil
	}

	err = cd.Unmarshal(b, object)
	if err != nil {
		internal.Log.Printf("cache: key=%q Unmarshal(%T) failed: %s", key, object, err)
		return err
	}

	return nil
}

func (cd *Codec) getBytes(key string, onlyLocalCache bool) ([]byte, error) {
	if cd.localCache != nil {
		b, ok := cd.localCache.Get(key)
		if ok {
			atomic.AddUint64(&cd.localHits, 1)
			return b, nil
		}
		atomic.AddUint64(&cd.localMisses, 1)
	}

	if onlyLocalCache {
		return nil, ErrCacheMiss
	}
	if cd.Redis == nil {
		if cd.localCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return nil, ErrCacheMiss
	}

	b, err := cd.Redis.Get(key).Bytes()
	if err != nil {
		atomic.AddUint64(&cd.misses, 1)
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		internal.Log.Printf("cache: Get key=%q failed: %s", key, err)
		return nil, err
	}
	atomic.AddUint64(&cd.hits, 1)

	if cd.localCache != nil {
		cd.localCache.Set(key, b)
	}
	return b, nil
}

// Once gets the item.Object for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Codec) Once(item *Item) error {
	b, cached, err := cd.getSetItemBytesOnce(item)
	if err != nil {
		return err
	}

	if item.Object == nil || len(b) == 0 {
		return nil
	}

	err = cd.Unmarshal(b, item.Object)
	if err != nil {
		internal.Log.Printf("cache: key=%q Unmarshal(%T) failed: %s", item.Key, item.Object, err)
		if cached {
			_ = cd.Delete(item.Key)
			return cd.Once(item)
		}
		return err
	}

	return nil
}

func (cd *Codec) getSetItemBytesOnce(item *Item) (b []byte, cached bool, err error) {
	if cd.localCache != nil {
		b, err := cd.getItemBytesFast(item)
		if err == nil {
			return b, true, nil
		}
	}

	obj, err := cd.group.Do(item.Key, func() (interface{}, error) {
		b, err := cd.getItemBytes(item)
		if err == nil {
			cached = true
			return b, nil
		}

		obj, err := item.Func()
		if err != nil {
			return nil, err
		}

		b, err = cd.setItem(&Item{
			Key:        item.Key,
			Object:     obj,
			Expiration: item.Expiration,
		})
		if b != nil {
			// Ignore error if we have the result.
			return b, nil
		}
		return nil, err
	})
	if err != nil {
		return nil, false, err
	}
	return obj.([]byte), cached, nil
}

func (cd *Codec) getItemBytes(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, false)
}

func (cd *Codec) getItemBytesFast(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, true)
}

func (cd *Codec) Delete(key string) error {
	return cd.DeleteContext(context.Background(), key)
}

func (cd *Codec) DeleteContext(ctx context.Context, key string) error {
	if cd.localCache != nil {
		cd.localCache.Delete(key)
	}

	if cd.Redis == nil {
		if cd.localCache == nil {
			return errRedisLocalCacheNil
		}
		return nil
	}

	deleted, err := cd.Redis.Del(key).Result()
	if err != nil {
		internal.Log.Printf("cache: Del key=%q failed: %s", key, err)
		return err
	}
	if deleted == 0 {
		return ErrCacheMiss
	}
	return nil
}

//------------------------------------------------------------------------------

type Stats struct {
	Hits        uint64
	Misses      uint64
	LocalHits   uint64
	LocalMisses uint64
}

// Stats returns cache statistics.
func (cd *Codec) Stats() *Stats {
	stats := Stats{
		Hits:   atomic.LoadUint64(&cd.hits),
		Misses: atomic.LoadUint64(&cd.misses),
	}
	if cd.localCache != nil {
		stats.LocalHits = atomic.LoadUint64(&cd.localHits)
		stats.LocalMisses = atomic.LoadUint64(&cd.localMisses)
	}
	return &stats
}
<commit_msg>Cleanup to not alloc Item<commit_after>package cache

import (
	"context"
	"errors"
	"sync/atomic"
	"time"

	"github.com/go-redis/cache/v7/internal"
	"github.com/go-redis/cache/v7/internal/lrucache"
	"github.com/go-redis/cache/v7/internal/singleflight"

	"github.com/go-redis/redis/v7"
)

var ErrCacheMiss = errors.New("cache: key is missing")
var errRedisLocalCacheNil = errors.New("cache: both Redis and LocalCache are nil")

func SetLogger(logger internal.Logger) {
	internal.Log = logger
}

type rediser interface {
	Set(key string, value interface{}, expiration time.Duration) *redis.StatusCmd
	Get(key string) *redis.StringCmd
	Del(keys ...string) *redis.IntCmd
}

type Item struct {
	Ctx context.Context

	Key    string
	Object interface{}

	// Func returns object to be cached.
	Func func() (interface{}, error)

	// Expiration is the cache expiration time.
	// Default expiration is 1 hour.
	Expiration time.Duration
}

func (item *Item) Context() context.Context {
	if item.Ctx == nil {
		return context.Background()
	}
	return item.Ctx
}

func (item *Item) object() (interface{}, error) {
	if item.Object != nil {
		return item.Object, nil
	}
	if item.Func != nil {
		return item.Func()
	}
	return nil, nil
}

func (item *Item) exp() time.Duration {
	if item.Expiration < 0 {
		return 0
	}
	if item.Expiration < time.Second {
		return time.Hour
	}
	return item.Expiration
}

type Codec struct {
	Redis rediser

	localCache *lrucache.Cache

	Marshal   func(interface{}) ([]byte, error)
	Unmarshal func([]byte, interface{}) error

	group singleflight.Group

	hits        uint64
	misses      uint64
	localHits   uint64
	localMisses uint64
}

// UseLocalCache causes Codec to cache items in local LRU cache.
func (cd *Codec) UseLocalCache(maxLen int, expiration time.Duration) {
	cd.localCache = lrucache.New(maxLen, expiration)
}

// Set caches the item.
func (cd *Codec) Set(item *Item) error {
	obj, err := item.object()
	if err != nil {
		return err
	}
	_, err = cd.set(item.Context(), item.Key, obj, item.exp())
	return err
}

func (cd *Codec) set(
	ctx context.Context, key string, obj interface{}, exp time.Duration,
) ([]byte, error) {
	b, err := cd.Marshal(obj)
	if err != nil {
		internal.Log.Printf("cache: Marshal key=%q failed: %s", key, err)
		return nil, err
	}

	if cd.localCache != nil {
		cd.localCache.Set(key, b)
	}

	if cd.Redis == nil {
		if cd.localCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return b, nil
	}

	err = cd.Redis.Set(key, b, exp).Err()
	if err != nil {
		internal.Log.Printf("cache: Set key=%q failed: %s", key, err)
	}
	return b, err
}

// Exists reports whether object for the given key exists.
func (cd *Codec) Exists(key string) bool {
	return cd.Get(key, nil) == nil
}

func (cd *Codec) ExistsContext(ctx context.Context, key string) bool {
	return cd.GetContext(ctx, key, nil) == nil
}

// Get gets the object for the given key.
func (cd *Codec) Get(key string, object interface{}) error {
	return cd.get(context.Background(), key, object, false)
}

func (cd *Codec) GetContext(ctx context.Context, key string, object interface{}) error {
	return cd.get(ctx, key, object, false)
}

func (cd *Codec) get(
	ctx context.Context, key string, object interface{}, onlyLocalCache bool,
) error {
	b, err := cd.getBytes(key, onlyLocalCache)
	if err != nil {
		return err
	}

	if object == nil || len(b) == 0 {
		return nil
	}

	err = cd.Unmarshal(b, object)
	if err != nil {
		internal.Log.Printf("cache: key=%q Unmarshal(%T) failed: %s", key, object, err)
		return err
	}

	return nil
}

func (cd *Codec) getBytes(key string, onlyLocalCache bool) ([]byte, error) {
	if cd.localCache != nil {
		b, ok := cd.localCache.Get(key)
		if ok {
			atomic.AddUint64(&cd.localHits, 1)
			return b, nil
		}
		atomic.AddUint64(&cd.localMisses, 1)
	}

	if onlyLocalCache {
		return nil, ErrCacheMiss
	}
	if cd.Redis == nil {
		if cd.localCache == nil {
			return nil, errRedisLocalCacheNil
		}
		return nil, ErrCacheMiss
	}

	b, err := cd.Redis.Get(key).Bytes()
	if err != nil {
		atomic.AddUint64(&cd.misses, 1)
		if err == redis.Nil {
			return nil, ErrCacheMiss
		}
		internal.Log.Printf("cache: Get key=%q failed: %s", key, err)
		return nil, err
	}
	atomic.AddUint64(&cd.hits, 1)

	if cd.localCache != nil {
		cd.localCache.Set(key, b)
	}
	return b, nil
}

// Once gets the item.Object for the given item.Key from the cache or
// executes, caches, and returns the results of the given item.Func,
// making sure that only one execution is in-flight for a given item.Key
// at a time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
func (cd *Codec) Once(item *Item) error {
	b, cached, err := cd.getSetItemBytesOnce(item)
	if err != nil {
		return err
	}

	if item.Object == nil || len(b) == 0 {
		return nil
	}

	err = cd.Unmarshal(b, item.Object)
	if err != nil {
		internal.Log.Printf("cache: key=%q Unmarshal(%T) failed: %s", item.Key, item.Object, err)
		if cached {
			_ = cd.Delete(item.Key)
			return cd.Once(item)
		}
		return err
	}

	return nil
}

func (cd *Codec) getSetItemBytesOnce(item *Item) (b []byte, cached bool, err error) {
	if cd.localCache != nil {
		b, err := cd.getItemBytesFast(item)
		if err == nil {
			return b, true, nil
		}
	}

	obj, err := cd.group.Do(item.Key, func() (interface{}, error) {
		b, err := cd.getItemBytes(item)
		if err == nil {
			cached = true
			return b, nil
		}

		obj, err := item.Func()
		if err != nil {
			return nil, err
		}

		b, err = cd.set(item.Context(), item.Key, obj, item.exp())
		if b != nil {
			// Ignore error if we have the result.
			return b, nil
		}
		return nil, err
	})
	if err != nil {
		return nil, false, err
	}
	return obj.([]byte), cached, nil
}

func (cd *Codec) getItemBytes(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, false)
}

func (cd *Codec) getItemBytesFast(item *Item) ([]byte, error) {
	return cd.getBytes(item.Key, true)
}

func (cd *Codec) Delete(key string) error {
	return cd.DeleteContext(context.Background(), key)
}

func (cd *Codec) DeleteContext(ctx context.Context, key string) error {
	if cd.localCache != nil {
		cd.localCache.Delete(key)
	}

	if cd.Redis == nil {
		if cd.localCache == nil {
			return errRedisLocalCacheNil
		}
		return nil
	}

	deleted, err := cd.Redis.Del(key).Result()
	if err != nil {
		internal.Log.Printf("cache: Del key=%q failed: %s", key, err)
		return err
	}
	if deleted == 0 {
		return ErrCacheMiss
	}
	return nil
}

//------------------------------------------------------------------------------

type Stats struct {
	Hits        uint64
	Misses      uint64
	LocalHits   uint64
	LocalMisses uint64
}

// Stats returns cache statistics.
func (cd *Codec) Stats() *Stats {
	stats := Stats{
		Hits:   atomic.LoadUint64(&cd.hits),
		Misses: atomic.LoadUint64(&cd.misses),
	}
	if cd.localCache != nil {
		stats.LocalHits = atomic.LoadUint64(&cd.localHits)
		stats.LocalMisses = atomic.LoadUint64(&cd.localMisses)
	}
	return &stats
}
<|endoftext|>


<commit_before>package llrb

import (
	"fmt"
	"testing"
)

func TestIsRed(t *testing.T) {
	if !isRed(NewNode(Int(5))) {
		t.Error("Expected Red")
	}
}

func TestInsert1(t *testing.T) {
	root := NewNode(String("S"))
	data := New(root)
	data.Insert(NewNode(String("E")))
	data.Insert(NewNode(String("A")))
	nd := data.Search(String("E"))
	if fmt.Sprintf("%v", nd.Key) != "E" {
		t.Errorf("Unexpected %v", nd.Key)
	}
	if fmt.Sprintf("%v", nd.Left.Key) != "A" {
		t.Errorf("Unexpected %v", nd.Left.Key)
	}
	if fmt.Sprintf("%v", nd.Right.Key) != "S" {
		t.Errorf("Unexpected %v", nd.Right.Key)
	}
	if !nd.Left.Black {
		t.Errorf("Left should be black but %v", nd.Left.Black)
	}
	if !nd.Right.Black {
		t.Errorf("Right should be black but %v", nd.Right.Black)
	}
}

func TestInsert2(t *testing.T) {
	root := NewNode(String("S"))
	data := New(root)
	data.Insert(NewNode(String("E")))
	data.Insert(NewNode(String("A")))
	data.Insert(NewNode(String("R")))
	data.Insert(NewNode(String("C")))
	data.Insert(NewNode(String("H")))
	data.Insert(NewNode(String("X")))
	data.Insert(NewNode(String("M")))
	data.Insert(NewNode(String("P")))
	data.Insert(NewNode(String("L")))
	nd := data.Search(String("E"))
	if fmt.Sprintf("%v", nd.Key) != "E" {
		t.Errorf("Unexpected %v", nd.Key)
	}
	if fmt.Sprintf("%v", nd.Left.Key) != "C" {
		t.Errorf("Unexpected %v", nd.Left.Key)
	}
	if fmt.Sprintf("%v", nd.Right.Key) != "L" {
		t.Errorf("Unexpected %v", nd.Right.Key)
	}
	if !nd.Left.Black {
		t.Errorf("Left should be black but %v", nd.Left.Black)
	}
	if nd.Left.Left.Black {
		t.Errorf("Left, Left should be red but %v", nd.Left.Left.Black)
	}
	if !nd.Right.Black {
		t.Errorf("Right should be black but %v", nd.Right.Black)
	}
	if nd.Right.Left.Black {
		t.Errorf("Right, Left should be red but %v", nd.Right.Black)
	}
	if fmt.Sprintf("%v", nd.Right.Left.Key) != "H" {
		t.Errorf("Unexpected %v", nd.Right.Left.Key)
	}
	if fmt.Sprintf("%v", data.Root.Key) != "M" {
		t.Errorf("Unexpected %v", data.Root.Key)
	}
	if fmt.Sprintf("%v", data.Root.Right.Right.Key) != "X" {
		t.Errorf("Unexpected %v", data.Root.Right.Right.Key)
	}
	if fmt.Sprintf("%v", data.Root.Right.Right.Left.Key) != "S" {
		t.Errorf("Unexpected %v", data.Root.Right.Right.Left.Key)
	}
	if fmt.Sprintf("%v", data.Root.Left.Key) != "E" {
		t.Errorf("Unexpected %v", data.Root.Left.Key)
	}
	if fmt.Sprintf("%v", data.Root.Right.Left.Key) != "P" {
		t.Errorf("Unexpected %v", data.Root.Right.Left.Key)
	}
}

func TestBalanceInsert1(t *testing.T) {
	root := NewNode(Float64(1))
	data := New(root)
	nums := []float64{3, 9, 13, 17, 20, 25, 39, 16, 15, 2, 2.5}
	for _, num := range nums {
		data.Insert(NewNode(Float64(num)))
	}
}

func TestBalanceInsert2(t *testing.T) {
	root := NewNode(Float64(1))
	data := New(root)
	nums := []float64{3, 9, 13, 17, 20, 25, 39, 16, 15, 2, 2.5}
	for _, num := range nums {
		data.Insert(NewNode(Float64(num)))
	}
}
<commit_msg>adding tests<commit_after>package llrb

import (
	"fmt"
	"testing"
)

func TestIsRed(t *testing.T) {
	if !isRed(NewNode(Int(5))) {
		t.Error("Expected Red")
	}
}

func TestInsert1(t *testing.T) {
	root := NewNode(String("S"))
	data := New(root)
	data.Insert(NewNode(String("E")))
	data.Insert(NewNode(String("A")))
	nd := data.Search(String("E"))
	if fmt.Sprintf("%v", nd.Key) != "E" {
		t.Errorf("Unexpected %v", nd.Key)
	}
	if fmt.Sprintf("%v", nd.Left.Key) != "A" {
		t.Errorf("Unexpected %v", nd.Left.Key)
	}
	if fmt.Sprintf("%v", nd.Right.Key) != "S" {
		t.Errorf("Unexpected %v", nd.Right.Key)
	}
	if !nd.Left.Black {
		t.Errorf("Left should be black but %v", nd.Left.Black)
	}
	if !nd.Right.Black {
		t.Errorf("Right should be black but %v", nd.Right.Black)
	}
}

func TestInsert2(t *testing.T) {
	root := NewNode(String("S"))
	data := New(root)
	data.Insert(NewNode(String("E")))
	data.Insert(NewNode(String("A")))
	data.Insert(NewNode(String("R")))
	data.Insert(NewNode(String("C")))
	data.Insert(NewNode(String("H")))
	data.Insert(NewNode(String("X")))
	data.Insert(NewNode(String("M")))
	data.Insert(NewNode(String("P")))
	data.Insert(NewNode(String("L")))
	nd := data.Search(String("E"))
	if fmt.Sprintf("%v", nd.Key) != "E" {
		t.Errorf("Unexpected %v", nd.Key)
	}
	if fmt.Sprintf("%v", nd.Left.Key) != "C" {
		t.Errorf("Unexpected %v", nd.Left.Key)
	}
	if fmt.Sprintf("%v", nd.Right.Key) != "L" {
		t.Errorf("Unexpected %v", nd.Right.Key)
	}
	if !nd.Left.Black {
		t.Errorf("Left should be black but %v", nd.Left.Black)
	}
	if nd.Left.Left.Black {
		t.Errorf("Left, Left should be red but %v", nd.Left.Left.Black)
	}
	if !nd.Right.Black {
		t.Errorf("Right should be black but %v", nd.Right.Black)
	}
	if nd.Right.Left.Black {
		t.Errorf("Right, Left should be red but %v", nd.Right.Black)
	}
	if fmt.Sprintf("%v", nd.Right.Left.Key) != "H" {
		t.Errorf("Unexpected %v", nd.Right.Left.Key)
	}
	if fmt.Sprintf("%v", data.Root.Key) != "M" {
		t.Errorf("Unexpected %v", data.Root.Key)
	}
	if fmt.Sprintf("%v", data.Root.Right.Right.Key) != "X" {
		t.Errorf("Unexpected %v", data.Root.Right.Right.Key)
	}
	if fmt.Sprintf("%v", data.Root.Right.Right.Left.Key) != "S" {
		t.Errorf("Unexpected %v", data.Root.Right.Right.Left.Key)
	}
	if fmt.Sprintf("%v", data.Root.Left.Key) != "E" {
		t.Errorf("Unexpected %v", data.Root.Left.Key)
	}
	if fmt.Sprintf("%v", data.Root.Right.Left.Key) != "P" {
		t.Errorf("Unexpected %v", data.Root.Right.Left.Key)
	}
}

func TestBalanceInsert1(t *testing.T) {
	root := NewNode(Float64(1))
	data := New(root)
	nums := []float64{3, 9, 13, 17, 20, 25, 39, 16, 15, 2, 2.5}
	for _, num := range nums {
		data.Insert(NewNode(Float64(num)))
	}
}

func TestBalanceInsert2(t *testing.T) {
	root := NewNode(Float64(1))
	data := New(root)
	nums := []float64{3, 9, 13, 17, 20, 25, 39, 16, 15, 2, 2.5}
	for _, num := range nums {
		data.Insert(NewNode(Float64(num)))

		t.Logf("Inserted: %f\n", num)
		if data.Search(Float64(num)) == nil {
			t.Fatal(num, "must not be nil")
		}
		if data.Search(Float64(1000.12)) != nil {
			t.Fatal(1000.12, "must be nil")
		}

		switch num {
		case 3:
			if data.Search(Float64(3)).Left.Key != Float64(1) {
				t.Fatal("3's Left child must be 1")
			}
			if data.Search(Float64(3)).Right != nil {
				t.Fatal("3's Right child must be nil")
			}
			if data.SearchParent(Float64(3)) != nil {
				t.Fatal("3's Parent must be nil")
			}
			if data.SearchParent(Float64(1)).Key != Float64(3) {
				t.Fatal("1's Parent must be 3")
			}
			if data.Search(Float64(1)).Right != nil {
				t.Fatal("1's Right child must be nil")
			}
			if data.Search(Float64(1)).Left != nil {
				t.Fatal("1's Left child must be nil")
			}

		case 9:

		case 13:

		case 17:

		case 20:

		case 25:

		case 39:

		case 16:

		case 15:

		case 2:

		case 2.5:

		default:
			t.Fatal(num, "shouldn't be there...")
		}
	}
}
<|endoftext|>