<commit_before>package nds

import (
	"bytes"
	"reflect"
	"sync"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/memcache"
)

// getMultiLimit is the App Engine datastore limit for the maximum number
// of entities that can be got by datastore.GetMulti at once.
// nds.GetMulti increases this limit by performing as many
// datastore.GetMulti as required concurrently and collating the results.
const getMultiLimit = 1000

// GetMulti works similar to datastore.GetMulti except for two important
// advantages:
//
// 1) It removes the API limit of 1000 entities per request by
// calling the datastore as many times as required to fetch all the keys. It
// does this efficiently and concurrently.
//
// 2) GetMulti function will automatically use memcache where possible before
// accssing the datastore. It uses a caching mechanism similar to the Python
// ndb package. However consistency is improved as NDB consistency issue
// http://goo.gl/3ByVlA is not an issue here or accessing the same key
// concurrently.
//
// If memcache is not working for any reason, GetMulti will default to using
// the datastore without compromising cache consistency.
//
// Important: If you use nds.GetMulti, you must also use the NDS put and delete
// functions in all your code touching the datastore to ensure data consistency.
// This includes using nds.RunInTransaction instead of
// datastore.RunInTransaction.
//
// Increase the datastore timeout if you get datastore_v3: TIMEOUT errors when
// getting thousands of entities. You can do this using
// http://godoc.org/code.google.com/p/appengine-go/appengine#Timeout.
//
// vals must be a []S, []*S, []I or []P, for some struct type S, some interface
// type I, or some non-interface non-pointer type P such that P or *P implements
// datastore.PropertyLoadSaver. If an []I, each element must be a valid dst for
// Get: it must be a struct pointer or implement datastore.PropertyLoadSaver.
//
// As a special case, datastore.PropertyList is an invalid type for dst, even
// though a PropertyList is a slice of structs. It is treated as invalid to
// avoid being mistakenly passed when []datastore.PropertyList was intended.
func GetMulti(c context.Context,
	keys []*datastore.Key, vals interface{}) error {

	v := reflect.ValueOf(vals)
	if err := checkMultiArgs(keys, v); err != nil {
		return err
	}

	if len(keys) == 0 {
		return nil
	}

	callCount := (len(keys)-1)/getMultiLimit + 1
	errs := make([]error, callCount)

	wg := sync.WaitGroup{}
	wg.Add(callCount)
	for i := 0; i < callCount; i++ {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}

		index := i
		keySlice := keys[lo:hi]
		valSlice := v.Slice(lo, hi)

		go func() {
			if _, ok := transactionFromContext(c); ok {
				errs[index] = datastoreGetMulti(c,
					keySlice, valSlice.Interface())
			} else {
				errs[index] = getMulti(c, keySlice, valSlice)
			}
			wg.Done()
		}()
	}
	wg.Wait()

	// Quick escape if all errors are nil.
	errsNil := true
	for _, err := range errs {
		if err != nil {
			errsNil = false
		}
	}
	if errsNil {
		return nil
	}

	groupedErrs := make(appengine.MultiError, len(keys))
	for i, err := range errs {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		if me, ok := err.(appengine.MultiError); ok {
			copy(groupedErrs[lo:hi], me)
		} else if err != nil {
			return err
		}
	}
	return groupedErrs
}

// Get loads the entity stored for key into val, which must be a struct pointer.
// Currently PropertyLoadSaver is not implemented. If there is no such entity
// for the key, Get returns ErrNoSuchEntity.
//
// The values of val's unmatched struct fields are not modified, and matching
// slice-typed fields are not reset before appending to them. In particular, it
// is recommended to pass a pointer to a zero valued struct on each Get call.
//
// ErrFieldMismatch is returned when a field is to be loaded into a different
// type than the one it was stored from, or when a field is missing or
// unexported in the destination struct. ErrFieldMismatch is only returned if
// val is a struct pointer.
func Get(c context.Context, key *datastore.Key, val interface{}) error {
	// GetMulti catches nil interface; we need to catch nil ptr here.
	if val == nil {
		return datastore.ErrInvalidEntityType
	}

	err := GetMulti(c, []*datastore.Key{key}, []interface{}{val})
	if me, ok := err.(appengine.MultiError); ok {
		return me[0]
	}
	return err
}

type cacheState byte

const (
	miss cacheState = iota
	internalLock
	externalLock
	done
)

type cacheItem struct {
	key         *datastore.Key
	memcacheKey string

	val reflect.Value
	err error

	item *memcache.Item

	state cacheState
}

// getMulti attempts to get entities from, memcache, then the datastore.
// datastore. It also tries to replenish memcache if needed available. It does
// this in such a way that GetMulti will never get stale results even if the
// function, datastore or server fails at any point. The caching strategy is
// borrowed from Python ndb with some improvements that eliminate some
// consistency issues surrounding ndb, including http://goo.gl/3ByVlA.
func getMulti(c context.Context,
	keys []*datastore.Key, vals reflect.Value) error {

	cacheItems := make([]cacheItem, len(keys))
	for i, key := range keys {
		cacheItems[i].key = key
		cacheItems[i].memcacheKey = createMemcacheKey(key)
		cacheItems[i].val = vals.Index(i)
		cacheItems[i].state = miss
	}

	memcacheCtx, err := memcacheContext(c)
	if err != nil {
		return err
	}

	loadMemcache(memcacheCtx, cacheItems)

	lockMemcache(memcacheCtx, cacheItems)

	if err := loadDatastore(c, cacheItems, vals.Type()); err != nil {
		return err
	}

	saveMemcache(memcacheCtx, cacheItems)

	me, errsNil := make(appengine.MultiError, len(cacheItems)), true
	for i, cacheItem := range cacheItems {
		if cacheItem.err != nil {
			me[i] = cacheItem.err
			errsNil = false
		}
	}

	if errsNil {
		return nil
	}
	return me
}

func loadMemcache(c context.Context, cacheItems []cacheItem) {

	memcacheKeys := make([]string, len(cacheItems))
	for i, cacheItem := range cacheItems {
		memcacheKeys[i] = cacheItem.memcacheKey
	}

	items, err := memcacheGetMulti(c, memcacheKeys)
	if err != nil {
		for i := range cacheItems {
			cacheItems[i].state = externalLock
		}
		log.Warningf(c, "nds:loadMemcache GetMulti %s", err)
		return
	}

	for i, memcacheKey := range memcacheKeys {
		if item, ok := items[memcacheKey]; ok {
			switch item.Flags {
			case lockItem:
				cacheItems[i].state = externalLock
			case noneItem:
				cacheItems[i].state = done
				cacheItems[i].err = datastore.ErrNoSuchEntity
			case entityItem:
				pl := datastore.PropertyList{}
				if err := unmarshal(item.Value, &pl); err != nil {
					log.Warningf(c, "nds:loadMemcache unmarshal %s", err)
					cacheItems[i].state = externalLock
					break
				}
				if err := setValue(cacheItems[i].val, pl); err == nil {
					cacheItems[i].state = done
				} else {
					log.Warningf(c, "nds:loadMemcache setValue %s", err)
					cacheItems[i].state = externalLock
				}
			default:
				log.Warningf(c, "nds:loadMemcache unknown item.Flags %d", item.Flags)
				cacheItems[i].state = externalLock
			}
		}
	}
}

func lockMemcache(c context.Context, cacheItems []cacheItem) {

	lockItems := make([]*memcache.Item, 0, len(cacheItems))
	lockMemcacheKeys := make([]string, 0, len(cacheItems))
	for i, cacheItem := range cacheItems {
		if cacheItem.state == miss {

			item := &memcache.Item{
				Key:        cacheItem.memcacheKey,
				Flags:      lockItem,
				Value:      itemLock(),
				Expiration: memcacheLockTime,
			}
			cacheItems[i].item = item
			lockItems = append(lockItems, item)
			lockMemcacheKeys = append(lockMemcacheKeys, cacheItem.memcacheKey)
		}
	}

	// We don't care if there are errors here.
	if err := memcacheAddMulti(c, lockItems); err != nil {
		log.Warningf(c, "nds:lockMemcache AddMulti %s", err)
	}

	// Get the items again so we can use CAS when updating the cache.
	items, err := memcacheGetMulti(c, lockMemcacheKeys)

	// Cache failed so forget about it and just use the datastore.
	if err != nil {
		for i, cacheItem := range cacheItems {
			if cacheItem.state == miss {
				cacheItems[i].state = externalLock
			}
		}
		log.Warningf(c, "nds:lockMemcache GetMulti %s", err)
		return
	}

	// Cache worked so figure out what items we got.
	for i, cacheItem := range cacheItems {
		if cacheItem.state == miss {
			if item, ok := items[cacheItem.memcacheKey]; ok {
				switch item.Flags {
				case lockItem:
					if bytes.Equal(item.Value, cacheItem.item.Value) {
						cacheItems[i].item = item
						cacheItems[i].state = internalLock
					} else {
						cacheItems[i].state = externalLock
					}
				case noneItem:
					cacheItems[i].state = done
					cacheItems[i].err = datastore.ErrNoSuchEntity
				case entityItem:
					pl := datastore.PropertyList{}
					if err := unmarshal(item.Value, &pl); err != nil {
						log.Warningf(c, "nds:lockMemcache unmarshal %s", err)
						cacheItems[i].state = externalLock
						break
					}
					if err := setValue(cacheItems[i].val, pl); err == nil {
						cacheItems[i].state = done
					} else {
						log.Warningf(c, "nds:lockMemcache setValue %s", err)
						cacheItems[i].state = externalLock
					}
				default:
					log.Warningf(c, "nds:lockMemcache unknown item.Flags %d",
						item.Flags)
					cacheItems[i].state = externalLock
				}
			} else {
				// We just added a memcache item but it now isn't available so
				// treat it as an extarnal lock.
				cacheItems[i].state = externalLock
			}
		}
	}
}

func loadDatastore(c context.Context, cacheItems []cacheItem,
	valsType reflect.Type) error {

	keys := make([]*datastore.Key, 0, len(cacheItems))
	vals := make([]datastore.PropertyList, 0, len(cacheItems))
	cacheItemsIndex := make([]int, 0, len(cacheItems))

	for i, cacheItem := range cacheItems {
		switch cacheItem.state {
		case internalLock, externalLock:
			keys = append(keys, cacheItem.key)
			vals = append(vals, datastore.PropertyList{})
			cacheItemsIndex = append(cacheItemsIndex, i)
		}
	}

	var me appengine.MultiError
	if err := datastoreGetMulti(c, keys, vals); err == nil {
		me = make(appengine.MultiError, len(keys))
	} else if e, ok := err.(appengine.MultiError); ok {
		me = e
	} else {
		return err
	}

	for i, index := range cacheItemsIndex {
		switch me[i] {
		case nil:
			pl := vals[i]
			val := cacheItems[index].val
			if err := setValue(val, pl); err != nil {
				return err
			}

			if cacheItems[index].state == internalLock {
				cacheItems[index].item.Flags = entityItem
				cacheItems[index].item.Expiration = 0
				if data, err := marshal(pl); err == nil {
					cacheItems[index].item.Value = data
				} else {
					cacheItems[index].state = externalLock
					log.Warningf(c, "nds:loadDatastore marshal %s", err)
				}
			}
		case datastore.ErrNoSuchEntity:
			if cacheItems[index].state == internalLock {
				cacheItems[index].item.Flags = noneItem
				cacheItems[index].item.Expiration = 0
				cacheItems[index].item.Value = []byte{}
			}
			cacheItems[index].err = datastore.ErrNoSuchEntity
		default:
			cacheItems[index].state = externalLock
			cacheItems[index].err = me[i]
		}
	}
	return nil
}

func saveMemcache(c context.Context, cacheItems []cacheItem) {

	saveItems := make([]*memcache.Item, 0, len(cacheItems))
	for _, cacheItem := range cacheItems {
		if cacheItem.state == internalLock {
			saveItems = append(saveItems, cacheItem.item)
		}
	}

	if err := memcacheCompareAndSwapMulti(c, saveItems); err != nil {
		log.Warningf(c, "nds:saveMemcache CompareAndSwapMulti %s", err)
	}
}
<commit_msg>Only spawn goroutine if necessary.<commit_after>package nds

import (
	"bytes"
	"reflect"
	"sync"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/memcache"
)

// getMultiLimit is the App Engine datastore limit for the maximum number
// of entities that can be got by datastore.GetMulti at once.
// nds.GetMulti increases this limit by performing as many
// datastore.GetMulti as required concurrently and collating the results.
const getMultiLimit = 1000

// GetMulti works similar to datastore.GetMulti except for two important
// advantages:
//
// 1) It removes the API limit of 1000 entities per request by
// calling the datastore as many times as required to fetch all the keys. It
// does this efficiently and concurrently.
//
// 2) GetMulti function will automatically use memcache where possible before
// accssing the datastore. It uses a caching mechanism similar to the Python
// ndb package. However consistency is improved as NDB consistency issue
// http://goo.gl/3ByVlA is not an issue here or accessing the same key
// concurrently.
//
// If memcache is not working for any reason, GetMulti will default to using
// the datastore without compromising cache consistency.
//
// Important: If you use nds.GetMulti, you must also use the NDS put and delete
// functions in all your code touching the datastore to ensure data consistency.
// This includes using nds.RunInTransaction instead of
// datastore.RunInTransaction.
//
// Increase the datastore timeout if you get datastore_v3: TIMEOUT errors when
// getting thousands of entities. You can do this using
// http://godoc.org/code.google.com/p/appengine-go/appengine#Timeout.
//
// vals must be a []S, []*S, []I or []P, for some struct type S, some interface
// type I, or some non-interface non-pointer type P such that P or *P implements
// datastore.PropertyLoadSaver. If an []I, each element must be a valid dst for
// Get: it must be a struct pointer or implement datastore.PropertyLoadSaver.
//
// As a special case, datastore.PropertyList is an invalid type for dst, even
// though a PropertyList is a slice of structs. It is treated as invalid to
// avoid being mistakenly passed when []datastore.PropertyList was intended.
func GetMulti(c context.Context,
	keys []*datastore.Key, vals interface{}) error {

	v := reflect.ValueOf(vals)
	if err := checkMultiArgs(keys, v); err != nil {
		return err
	}

	if len(keys) == 0 {
		return nil
	}

	wg := sync.WaitGroup{}

	callCount := (len(keys)-1)/getMultiLimit + 1
	errs := make([]error, callCount)
	call := func(i int, keys []*datastore.Key, vals reflect.Value) {
		if _, ok := transactionFromContext(c); ok {
			errs[i] = datastoreGetMulti(c, keys, vals.Interface())
		} else {
			errs[i] = getMulti(c, keys, vals)
		}
		wg.Done()
	}

	wg.Add(callCount)
	for i := 0; i < callCount; i++ {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}

		if i == callCount-1 {
			call(i, keys[lo:hi], v.Slice(lo, hi))
		} else {
			go call(i, keys[lo:hi], v.Slice(lo, hi))
		}
	}
	wg.Wait()

	// Quick escape if all errors are nil.
	errsNil := true
	for _, err := range errs {
		if err != nil {
			errsNil = false
		}
	}
	if errsNil {
		return nil
	}

	groupedErrs := make(appengine.MultiError, len(keys))
	for i, err := range errs {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		if me, ok := err.(appengine.MultiError); ok {
			copy(groupedErrs[lo:hi], me)
		} else if err != nil {
			return err
		}
	}
	return groupedErrs
}

// Get loads the entity stored for key into val, which must be a struct pointer.
// Currently PropertyLoadSaver is not implemented. If there is no such entity
// for the key, Get returns ErrNoSuchEntity.
//
// The values of val's unmatched struct fields are not modified, and matching
// slice-typed fields are not reset before appending to them. In particular, it
// is recommended to pass a pointer to a zero valued struct on each Get call.
//
// ErrFieldMismatch is returned when a field is to be loaded into a different
// type than the one it was stored from, or when a field is missing or
// unexported in the destination struct. ErrFieldMismatch is only returned if
// val is a struct pointer.
func Get(c context.Context, key *datastore.Key, val interface{}) error {
	// GetMulti catches nil interface; we need to catch nil ptr here.
	if val == nil {
		return datastore.ErrInvalidEntityType
	}

	err := GetMulti(c, []*datastore.Key{key}, []interface{}{val})
	if me, ok := err.(appengine.MultiError); ok {
		return me[0]
	}
	return err
}

type cacheState byte

const (
	miss cacheState = iota
	internalLock
	externalLock
	done
)

type cacheItem struct {
	key         *datastore.Key
	memcacheKey string

	val reflect.Value
	err error

	item *memcache.Item

	state cacheState
}

// getMulti attempts to get entities from, memcache, then the datastore.
// datastore. It also tries to replenish memcache if needed available. It does
// this in such a way that GetMulti will never get stale results even if the
// function, datastore or server fails at any point. The caching strategy is
// borrowed from Python ndb with some improvements that eliminate some
// consistency issues surrounding ndb, including http://goo.gl/3ByVlA.
func getMulti(c context.Context,
	keys []*datastore.Key, vals reflect.Value) error {

	cacheItems := make([]cacheItem, len(keys))
	for i, key := range keys {
		cacheItems[i].key = key
		cacheItems[i].memcacheKey = createMemcacheKey(key)
		cacheItems[i].val = vals.Index(i)
		cacheItems[i].state = miss
	}

	memcacheCtx, err := memcacheContext(c)
	if err != nil {
		return err
	}

	loadMemcache(memcacheCtx, cacheItems)

	lockMemcache(memcacheCtx, cacheItems)

	if err := loadDatastore(c, cacheItems, vals.Type()); err != nil {
		return err
	}

	saveMemcache(memcacheCtx, cacheItems)

	me, errsNil := make(appengine.MultiError, len(cacheItems)), true
	for i, cacheItem := range cacheItems {
		if cacheItem.err != nil {
			me[i] = cacheItem.err
			errsNil = false
		}
	}

	if errsNil {
		return nil
	}
	return me
}

func loadMemcache(c context.Context, cacheItems []cacheItem) {

	memcacheKeys := make([]string, len(cacheItems))
	for i, cacheItem := range cacheItems {
		memcacheKeys[i] = cacheItem.memcacheKey
	}

	items, err := memcacheGetMulti(c, memcacheKeys)
	if err != nil {
		for i := range cacheItems {
			cacheItems[i].state = externalLock
		}
		log.Warningf(c, "nds:loadMemcache GetMulti %s", err)
		return
	}

	for i, memcacheKey := range memcacheKeys {
		if item, ok := items[memcacheKey]; ok {
			switch item.Flags {
			case lockItem:
				cacheItems[i].state = externalLock
			case noneItem:
				cacheItems[i].state = done
				cacheItems[i].err = datastore.ErrNoSuchEntity
			case entityItem:
				pl := datastore.PropertyList{}
				if err := unmarshal(item.Value, &pl); err != nil {
					log.Warningf(c, "nds:loadMemcache unmarshal %s", err)
					cacheItems[i].state = externalLock
					break
				}
				if err := setValue(cacheItems[i].val, pl); err == nil {
					cacheItems[i].state = done
				} else {
					log.Warningf(c, "nds:loadMemcache setValue %s", err)
					cacheItems[i].state = externalLock
				}
			default:
				log.Warningf(c, "nds:loadMemcache unknown item.Flags %d", item.Flags)
				cacheItems[i].state = externalLock
			}
		}
	}
}

func lockMemcache(c context.Context, cacheItems []cacheItem) {

	lockItems := make([]*memcache.Item, 0, len(cacheItems))
	lockMemcacheKeys := make([]string, 0, len(cacheItems))
	for i, cacheItem := range cacheItems {
		if cacheItem.state == miss {

			item := &memcache.Item{
				Key:        cacheItem.memcacheKey,
				Flags:      lockItem,
				Value:      itemLock(),
				Expiration: memcacheLockTime,
			}
			cacheItems[i].item = item
			lockItems = append(lockItems, item)
			lockMemcacheKeys = append(lockMemcacheKeys, cacheItem.memcacheKey)
		}
	}

	// We don't care if there are errors here.
	if err := memcacheAddMulti(c, lockItems); err != nil {
		log.Warningf(c, "nds:lockMemcache AddMulti %s", err)
	}

	// Get the items again so we can use CAS when updating the cache.
	items, err := memcacheGetMulti(c, lockMemcacheKeys)

	// Cache failed so forget about it and just use the datastore.
	if err != nil {
		for i, cacheItem := range cacheItems {
			if cacheItem.state == miss {
				cacheItems[i].state = externalLock
			}
		}
		log.Warningf(c, "nds:lockMemcache GetMulti %s", err)
		return
	}

	// Cache worked so figure out what items we got.
	for i, cacheItem := range cacheItems {
		if cacheItem.state == miss {
			if item, ok := items[cacheItem.memcacheKey]; ok {
				switch item.Flags {
				case lockItem:
					if bytes.Equal(item.Value, cacheItem.item.Value) {
						cacheItems[i].item = item
						cacheItems[i].state = internalLock
					} else {
						cacheItems[i].state = externalLock
					}
				case noneItem:
					cacheItems[i].state = done
					cacheItems[i].err = datastore.ErrNoSuchEntity
				case entityItem:
					pl := datastore.PropertyList{}
					if err := unmarshal(item.Value, &pl); err != nil {
						log.Warningf(c, "nds:lockMemcache unmarshal %s", err)
						cacheItems[i].state = externalLock
						break
					}
					if err := setValue(cacheItems[i].val, pl); err == nil {
						cacheItems[i].state = done
					} else {
						log.Warningf(c, "nds:lockMemcache setValue %s", err)
						cacheItems[i].state = externalLock
					}
				default:
					log.Warningf(c, "nds:lockMemcache unknown item.Flags %d",
						item.Flags)
					cacheItems[i].state = externalLock
				}
			} else {
				// We just added a memcache item but it now isn't available so
				// treat it as an extarnal lock.
				cacheItems[i].state = externalLock
			}
		}
	}
}

func loadDatastore(c context.Context, cacheItems []cacheItem,
	valsType reflect.Type) error {

	keys := make([]*datastore.Key, 0, len(cacheItems))
	vals := make([]datastore.PropertyList, 0, len(cacheItems))
	cacheItemsIndex := make([]int, 0, len(cacheItems))

	for i, cacheItem := range cacheItems {
		switch cacheItem.state {
		case internalLock, externalLock:
			keys = append(keys, cacheItem.key)
			vals = append(vals, datastore.PropertyList{})
			cacheItemsIndex = append(cacheItemsIndex, i)
		}
	}

	var me appengine.MultiError
	if err := datastoreGetMulti(c, keys, vals); err == nil {
		me = make(appengine.MultiError, len(keys))
	} else if e, ok := err.(appengine.MultiError); ok {
		me = e
	} else {
		return err
	}

	for i, index := range cacheItemsIndex {
		switch me[i] {
		case nil:
			pl := vals[i]
			val := cacheItems[index].val
			if err := setValue(val, pl); err != nil {
				return err
			}

			if cacheItems[index].state == internalLock {
				cacheItems[index].item.Flags = entityItem
				cacheItems[index].item.Expiration = 0
				if data, err := marshal(pl); err == nil {
					cacheItems[index].item.Value = data
				} else {
					cacheItems[index].state = externalLock
					log.Warningf(c, "nds:loadDatastore marshal %s", err)
				}
			}
		case datastore.ErrNoSuchEntity:
			if cacheItems[index].state == internalLock {
				cacheItems[index].item.Flags = noneItem
				cacheItems[index].item.Expiration = 0
				cacheItems[index].item.Value = []byte{}
			}
			cacheItems[index].err = datastore.ErrNoSuchEntity
		default:
			cacheItems[index].state = externalLock
			cacheItems[index].err = me[i]
		}
	}
	return nil
}

func saveMemcache(c context.Context, cacheItems []cacheItem) {

	saveItems := make([]*memcache.Item, 0, len(cacheItems))
	for _, cacheItem := range cacheItems {
		if cacheItem.state == internalLock {
			saveItems = append(saveItems, cacheItem.item)
		}
	}

	if err := memcacheCompareAndSwapMulti(c, saveItems); err != nil {
		log.Warningf(c, "nds:saveMemcache CompareAndSwapMulti %s", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	// "fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

const GH = "https://github.com/"

func sanitize(bundle string) string {
	return strings.Replace(bundle, "/", "-", -1)
}

func folder(bundle string, home string) string {
	return home + sanitize(bundle)
}

func Clone(bundle string, home string) (string, error) {
	folder := folder(bundle, home)
	if _, err := os.Stat(folder); os.IsNotExist(err) {
		clone := exec.Command("git", "clone", "--depth", "1", GH+bundle, folder)
		return folder, clone.Run()
	}
	return folder, nil
}

func Pull(bundle string, home string) (string, error) {
	folder := folder(bundle, home)
	pull := exec.Command("git", "-C", folder, "pull", "origin", "master")
	err := pull.Run()
	return folder, err
}

func Update(home string) ([]string, error) {
	bundles, _ := ioutil.ReadDir(home)
	var sourceables []string
	for _, bundle := range bundles {
		if bundle.Mode().IsDir() && bundle.Name()[0] != '.' {
			updated, err := Pull(bundle.Name(), home)
			if err != nil {
				return sourceables, err
			}
			sourceables = append(sourceables, updated)
		}
	}
	return sourceables, nil
}
<commit_msg>cleanup<commit_after>package main

import (
	// "fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

const GH = "https://github.com/"

func folder(bundle string, home string) string {
	return home + strings.Replace(bundle, "/", "-", -1)
}

func Clone(bundle string, home string) (string, error) {
	folder := folder(bundle, home)
	if _, err := os.Stat(folder); os.IsNotExist(err) {
		clone := exec.Command("git", "clone", "--depth", "1", GH+bundle, folder)
		return folder, clone.Run()
	}
	return folder, nil
}

func Pull(bundle string, home string) (string, error) {
	folder := folder(bundle, home)
	pull := exec.Command("git", "-C", folder, "pull", "origin", "master")
	err := pull.Run()
	return folder, err
}

func Update(home string) ([]string, error) {
	bundles, _ := ioutil.ReadDir(home)
	var sourceables []string
	for _, bundle := range bundles {
		if bundle.Mode().IsDir() && bundle.Name()[0] != '.' {
			updated, err := Pull(bundle.Name(), home)
			if err != nil {
				return sourceables, err
			}
			sourceables = append(sourceables, updated)
		}
	}
	return sourceables, nil
}
<|endoftext|>


<commit_before>package auth

import (
	"context"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"strings"

	"github.com/micro/go-micro/v2/api/resolver"
	"github.com/micro/go-micro/v2/api/resolver/path"
	"github.com/micro/go-micro/v2/auth"
	"github.com/micro/go-micro/v2/logger"
	"golang.org/x/net/publicsuffix"
)

// CombinedAuthHandler wraps a server and authenticates requests
func CombinedAuthHandler(prefix, namespace string, r resolver.Resolver, h http.Handler) http.Handler {
	if r == nil {
		r = path.NewResolver()
	}

	return authHandler{
		handler:       h,
		resolver:      r,
		auth:          auth.DefaultAuth,
		servicePrefix: prefix,
		namespace:     namespace,
	}
}

type authHandler struct {
	handler       http.Handler
	auth          auth.Auth
	resolver      resolver.Resolver
	namespace     string
	servicePrefix string
}

func (h authHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Determine the namespace and set it in the header
	namespace := h.NamespaceFromRequest(req)
	req.Header.Set(auth.NamespaceKey, namespace)

	// Extract the token from the request
	var token string
	if header := req.Header.Get("Authorization"); len(header) > 0 {
		// Extract the auth token from the request
		if strings.HasPrefix(header, auth.BearerScheme) {
			token = header[len(auth.BearerScheme):]
		}
	} else {
		// Get the token out the cookies if not provided in headers
		if c, err := req.Cookie("micro-token"); err == nil && c != nil {
			token = strings.TrimPrefix(c.Value, auth.TokenCookieName+"=")
			req.Header.Set("Authorization", auth.BearerScheme+token)
		}
	}

	// Get the account using the token, fallback to a blank account
	// since some endpoints can be unauthenticated, so the lack of an
	// account doesn't necesserially mean a forbidden request
	acc, err := h.auth.Inspect(token)
	if err != nil {
		acc = &auth.Account{Namespace: namespace}
	}

	// Check the accounts namespace matches the namespace we're operating
	// within. If not forbid the request and log the occurance.
	if acc.Namespace != namespace {
		logger.Warnf("Cross namespace request forbidden: account %v (%v) requested access to %v in the %v namespace", acc.ID, acc.Namespace, req.URL.Path, namespace)
		http.Error(w, "Forbidden namespace", 403)
	}

	// Determine the name of the service being requested
	endpoint, err := h.resolver.Resolve(req)
	if err == resolver.ErrInvalidPath || err == resolver.ErrNotFound {
		// a file not served by the resolver has been requested (e.g. favicon.ico)
		endpoint = &resolver.Endpoint{Path: req.URL.Path}
	} else if err != nil {
		logger.Error(err)
		http.Error(w, err.Error(), 500)
		return
	} else {
		// set the endpoint in the context so it can be used to resolve
		// the request later
		ctx := context.WithValue(req.Context(), resolver.Endpoint{}, endpoint)
		*req = *req.Clone(ctx)
	}

	// construct the resource name, e.g. home => go.micro.web.home
	resName := h.servicePrefix
	if len(endpoint.Name) > 0 {
		resName = resName + "." + endpoint.Name
	}

	// determine the resource path. there is an inconsistency in how resolvers
	// use method, some use it as Users.ReadUser (the rpc method), and others
	// use it as the HTTP method, e.g GET. TODO: Refactor this to make it consistent.
	resEndpoint := endpoint.Path
	if len(endpoint.Path) == 0 {
		resEndpoint = endpoint.Method
	}

	// Perform the verification check to see if the account has access to
	// the resource they're requesting
	res := &auth.Resource{Type: "service", Name: resName, Endpoint: resEndpoint, Namespace: namespace}
	if err := h.auth.Verify(acc, res); err == nil {
		// The account has the necessary permissions to access the resource
		h.handler.ServeHTTP(w, req)
		return
	}

	// The account is set, but they don't have enough permissions, hence
	// we return a forbidden error.
	if len(acc.ID) > 0 {
		http.Error(w, "Forbidden request", 403)
		return
	}

	// If there is no auth login url set, 401
	loginURL := h.auth.Options().LoginURL
	if loginURL == "" {
		http.Error(w, "unauthorized request", 401)
		return
	}

	// Redirect to the login path
	params := url.Values{"redirect_to": {req.URL.Path}}
	loginWithRedirect := fmt.Sprintf("%v?%v", loginURL, params.Encode())
	http.Redirect(w, req, loginWithRedirect, http.StatusTemporaryRedirect)
}

func (h authHandler) NamespaceFromRequest(req *http.Request) string {
	// check to see what the provided namespace is, we only do
	// domain mapping if the namespace is set to 'domain'
	if h.namespace != "domain" {
		return h.namespace
	}

	// determine the host, e.g. dev.micro.mu:8080
	var host string
	if h, _, err := net.SplitHostPort(req.Host); err == nil {
		host = h // host does contain a port
	} else if strings.Contains(err.Error(), "missing port in address") {
		host = req.Host // host does not contain a port
	}

	// check for an ip address
	if net.ParseIP(host) != nil {
		return auth.DefaultNamespace
	}

	// check for dev enviroment
	if host == "localhost" || host == "127.0.0.1" {
		return auth.DefaultNamespace
	}

	// extract the top level domain plus one (e.g. 'myapp.com')
	domain, err := publicsuffix.EffectiveTLDPlusOne(host)
	if err != nil {
		logger.Debugf("Unable to extract domain from %v", host)
		return auth.DefaultNamespace
	}

	// check to see if the domain matches the host of micr.mu, in
	// these casees we return the default namespace
	if domain == host || domain == "micro.mu" {
		return auth.DefaultNamespace
	}

	// remove the domain from the host, leaving the subdomain
	subdomain := strings.TrimSuffix(host, "."+domain)

	// return the reversed subdomain as the namespace
	comps := strings.Split(subdomain, ".")
	for i := len(comps)/2 - 1; i >= 0; i-- {
		opp := len(comps) - 1 - i
		comps[i], comps[opp] = comps[opp], comps[i]
	}
	return strings.Join(comps, ".")
}
<commit_msg>Fix comments typo<commit_after>package auth

import (
	"context"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"strings"

	"github.com/micro/go-micro/v2/api/resolver"
	"github.com/micro/go-micro/v2/api/resolver/path"
	"github.com/micro/go-micro/v2/auth"
	"github.com/micro/go-micro/v2/logger"
	"golang.org/x/net/publicsuffix"
)

// CombinedAuthHandler wraps a server and authenticates requests
func CombinedAuthHandler(prefix, namespace string, r resolver.Resolver, h http.Handler) http.Handler {
	if r == nil {
		r = path.NewResolver()
	}

	return authHandler{
		handler:       h,
		resolver:      r,
		auth:          auth.DefaultAuth,
		servicePrefix: prefix,
		namespace:     namespace,
	}
}

type authHandler struct {
	handler       http.Handler
	auth          auth.Auth
	resolver      resolver.Resolver
	namespace     string
	servicePrefix string
}

func (h authHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Determine the namespace and set it in the header
	namespace := h.NamespaceFromRequest(req)
	req.Header.Set(auth.NamespaceKey, namespace)

	// Extract the token from the request
	var token string
	if header := req.Header.Get("Authorization"); len(header) > 0 {
		// Extract the auth token from the request
		if strings.HasPrefix(header, auth.BearerScheme) {
			token = header[len(auth.BearerScheme):]
		}
	} else {
		// Get the token out the cookies if not provided in headers
		if c, err := req.Cookie("micro-token"); err == nil && c != nil {
			token = strings.TrimPrefix(c.Value, auth.TokenCookieName+"=")
			req.Header.Set("Authorization", auth.BearerScheme+token)
		}
	}

	// Get the account using the token, fallback to a blank account
	// since some endpoints can be unauthenticated, so the lack of an
	// account doesn't necesserially mean a forbidden request
	acc, err := h.auth.Inspect(token)
	if err != nil {
		acc = &auth.Account{Namespace: namespace}
	}

	// Check the accounts namespace matches the namespace we're operating
	// within. If not forbid the request and log the occurance.
	if acc.Namespace != namespace {
		logger.Warnf("Cross namespace request forbidden: account %v (%v) requested access to %v in the %v namespace", acc.ID, acc.Namespace, req.URL.Path, namespace)
		http.Error(w, "Forbidden namespace", 403)
	}

	// Determine the name of the service being requested
	endpoint, err := h.resolver.Resolve(req)
	if err == resolver.ErrInvalidPath || err == resolver.ErrNotFound {
		// a file not served by the resolver has been requested (e.g. favicon.ico)
		endpoint = &resolver.Endpoint{Path: req.URL.Path}
	} else if err != nil {
		logger.Error(err)
		http.Error(w, err.Error(), 500)
		return
	} else {
		// set the endpoint in the context so it can be used to resolve
		// the request later
		ctx := context.WithValue(req.Context(), resolver.Endpoint{}, endpoint)
		*req = *req.Clone(ctx)
	}

	// construct the resource name, e.g. home => go.micro.web.home
	resName := h.servicePrefix
	if len(endpoint.Name) > 0 {
		resName = resName + "." + endpoint.Name
	}

	// determine the resource path. there is an inconsistency in how resolvers
	// use method, some use it as Users.ReadUser (the rpc method), and others
	// use it as the HTTP method, e.g GET. TODO: Refactor this to make it consistent.
	resEndpoint := endpoint.Path
	if len(endpoint.Path) == 0 {
		resEndpoint = endpoint.Method
	}

	// Perform the verification check to see if the account has access to
	// the resource they're requesting
	res := &auth.Resource{Type: "service", Name: resName, Endpoint: resEndpoint, Namespace: namespace}
	if err := h.auth.Verify(acc, res); err == nil {
		// The account has the necessary permissions to access the resource
		h.handler.ServeHTTP(w, req)
		return
	}

	// The account is set, but they don't have enough permissions, hence
	// we return a forbidden error.
	if len(acc.ID) > 0 {
		http.Error(w, "Forbidden request", 403)
		return
	}

	// If there is no auth login url set, 401
	loginURL := h.auth.Options().LoginURL
	if loginURL == "" {
		http.Error(w, "unauthorized request", 401)
		return
	}

	// Redirect to the login path
	params := url.Values{"redirect_to": {req.URL.Path}}
	loginWithRedirect := fmt.Sprintf("%v?%v", loginURL, params.Encode())
	http.Redirect(w, req, loginWithRedirect, http.StatusTemporaryRedirect)
}

func (h authHandler) NamespaceFromRequest(req *http.Request) string {
	// check to see what the provided namespace is, we only do
	// domain mapping if the namespace is set to 'domain'
	if h.namespace != "domain" {
		return h.namespace
	}

	// determine the host, e.g. dev.micro.mu:8080
	var host string
	if h, _, err := net.SplitHostPort(req.Host); err == nil {
		host = h // host does contain a port
	} else if strings.Contains(err.Error(), "missing port in address") {
		host = req.Host // host does not contain a port
	}

	// check for an ip address
	if net.ParseIP(host) != nil {
		return auth.DefaultNamespace
	}

	// check for dev enviroment
	if host == "localhost" || host == "127.0.0.1" {
		return auth.DefaultNamespace
	}

	// extract the top level domain plus one (e.g. 'myapp.com')
	domain, err := publicsuffix.EffectiveTLDPlusOne(host)
	if err != nil {
		logger.Debugf("Unable to extract domain from %v", host)
		return auth.DefaultNamespace
	}

	// check to see if the domain matches the host of micro.mu, in
	// these cases we return the default namespace
	if domain == host || domain == "micro.mu" {
		return auth.DefaultNamespace
	}

	// remove the domain from the host, leaving the subdomain
	subdomain := strings.TrimSuffix(host, "."+domain)

	// return the reversed subdomain as the namespace
	comps := strings.Split(subdomain, ".")
	for i := len(comps)/2 - 1; i >= 0; i-- {
		opp := len(comps) - 1 - i
		comps[i], comps[opp] = comps[opp], comps[i]
	}
	return strings.Join(comps, ".")
}
<|endoftext|>


<commit_before>package consensus

import (
	"encoding/binary"
	"testing"
	"time"

	abci "github.com/tendermint/abci/types"
	"github.com/tendermint/tendermint/types"

	. "github.com/tendermint/tmlibs/common"
)

func init() {
	config = ResetConfig("consensus_mempool_test")
}

func TestNoProgressUntilTxsAvailable(t *testing.T) {
	config := ResetConfig("consensus_mempool_txs_available_test")
	config.Consensus.NoEmptyBlocks = true
	state, privVals := randGenesisState(1, false, 10)
	cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication())
	cs.mempool.EnableTxsAvailable()
	height, round := cs.Height, cs.Round
	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
	startTestRound(cs, height, round)

	ensureNewStep(newBlockCh) // first block gets committed
	ensureNoNewStep(newBlockCh)
	deliverTxsRange(cs, 0, 2)
	ensureNewStep(newBlockCh) // commit txs
	ensureNewStep(newBlockCh) // commit updated app hash
	ensureNoNewStep(newBlockCh)
}

func deliverTxsRange(cs *ConsensusState, start, end int) {
	// Deliver some txs.
	for i := start; i < end; i++ {
		txBytes := make([]byte, 8)
		binary.BigEndian.PutUint64(txBytes, uint64(i))
		err := cs.mempool.CheckTx(txBytes, nil)
		if err != nil {
			panic(Fmt("Error after CheckTx: %v", err))
		}
	}
}

func TestTxConcurrentWithCommit(t *testing.T) {

	state, privVals := randGenesisState(1, false, 10)
	cs := newConsensusState(state, privVals[0], NewCounterApplication())
	height, round := cs.Height, cs.Round
	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)

	NTxs := 10000
	go deliverTxsRange(cs, 0, NTxs)

	startTestRound(cs, height, round)
	ticker := time.NewTicker(time.Second * 20)
	for nTxs := 0; nTxs < NTxs; {
		select {
		case b := <-newBlockCh:
			nTxs += b.(types.TMEventData).Unwrap().(types.EventDataNewBlock).Block.Header.NumTxs
		case <-ticker.C:
			panic("Timed out waiting to commit blocks with transactions")
		}
	}
}

func TestRmBadTx(t *testing.T) {
	state, privVals := randGenesisState(1, false, 10)
	app := NewCounterApplication()
	cs := newConsensusState(state, privVals[0], app)

	// increment the counter by 1
	txBytes := make([]byte, 8)
	binary.BigEndian.PutUint64(txBytes, uint64(0))
	app.DeliverTx(txBytes)
	app.Commit()

	ch := make(chan struct{})
	cbCh := make(chan struct{})
	go func() {
		// Try to send the tx through the mempool.
		// CheckTx should not err, but the app should return a bad abci code
		// and the tx should get removed from the pool
		err := cs.mempool.CheckTx(txBytes, func(r *abci.Response) {
			if r.GetCheckTx().Code != abci.CodeType_BadNonce {
				t.Fatalf("expected checktx to return bad nonce, got %v", r)
			}
			cbCh <- struct{}{}
		})
		if err != nil {
			t.Fatal("Error after CheckTx: %v", err)
		}

		// check for the tx
		for {
			time.Sleep(time.Second)
			txs := cs.mempool.Reap(1)
			if len(txs) == 0 {
				ch <- struct{}{}
				return
			}

		}
	}()

	// Wait until the tx returns
	ticker := time.After(time.Second * 5)
	select {
	case <-cbCh:
		// success
	case <-ticker:
		t.Fatalf("Timed out waiting for tx to return")
	}

	// Wait until the tx is removed
	ticker = time.After(time.Second * 5)
	select {
	case <-ch:
		// success
	case <-ticker:
		t.Fatalf("Timed out waiting for tx to be removed")
	}
}

// CounterApplication that maintains a mempool state and resets it upon commit
type CounterApplication struct {
	abci.BaseApplication

	txCount        int
	mempoolTxCount int
}

func NewCounterApplication() *CounterApplication {
	return &CounterApplication{}
}

func (app *CounterApplication) Info() abci.ResponseInfo {
	return abci.ResponseInfo{Data: Fmt("txs:%v", app.txCount)}
}

func (app *CounterApplication) DeliverTx(tx []byte) abci.Result {
	return runTx(tx, &app.txCount)
}

func (app *CounterApplication) CheckTx(tx []byte) abci.Result {
	return runTx(tx, &app.mempoolTxCount)
}

func runTx(tx []byte, countPtr *int) abci.Result {
	count := *countPtr
	tx8 := make([]byte, 8)
	copy(tx8[len(tx8)-len(tx):], tx)
	txValue := binary.BigEndian.Uint64(tx8)
	if txValue != uint64(count) {
		return abci.ErrBadNonce.AppendLog(Fmt("Invalid nonce. Expected %v, got %v", count, txValue))
	}
	*countPtr += 1
	return abci.OK
}

func (app *CounterApplication) Commit() abci.Result {
	app.mempoolTxCount = app.txCount
	if app.txCount == 0 {
		return abci.OK
	} else {
		hash := make([]byte, 8)
		binary.BigEndian.PutUint64(hash, uint64(app.txCount))
		return abci.NewResultOK(hash, "")
	}
}
<commit_msg>test progress in higher round<commit_after>package consensus

import (
	"encoding/binary"
	"testing"
	"time"

	abci "github.com/tendermint/abci/types"
	"github.com/tendermint/tendermint/types"

	. "github.com/tendermint/tmlibs/common"
)

func init() {
	config = ResetConfig("consensus_mempool_test")
}

func TestNoProgressUntilTxsAvailable(t *testing.T) {
	config := ResetConfig("consensus_mempool_txs_available_test")
	config.Consensus.NoEmptyBlocks = true
	state, privVals := randGenesisState(1, false, 10)
	cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication())
	cs.mempool.EnableTxsAvailable()
	height, round := cs.Height, cs.Round
	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
	startTestRound(cs, height, round)

	ensureNewStep(newBlockCh) // first block gets committed
	ensureNoNewStep(newBlockCh)
	deliverTxsRange(cs, 0, 2)
	ensureNewStep(newBlockCh) // commit txs
	ensureNewStep(newBlockCh) // commit updated app hash
	ensureNoNewStep(newBlockCh)

}

func TestProgressInHigherRound(t *testing.T) {
	config := ResetConfig("consensus_mempool_txs_available_test")
	config.Consensus.NoEmptyBlocks = true
	state, privVals := randGenesisState(1, false, 10)
	cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication())
	cs.mempool.EnableTxsAvailable()
	height, round := cs.Height, cs.Round
	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)
	newRoundCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewRound(), 1)
	timeoutCh := subscribeToEvent(cs.evsw, "tester", types.EventStringTimeoutPropose(), 1)
	cs.setProposal = func(proposal *types.Proposal) error {
		if cs.Height == 2 && cs.Round == 0 {
			// dont set the proposal in round 0 so we timeout and
			// go to next round
			cs.Logger.Info("Ignoring set proposal at height 2, round 0")
			return nil
		}
		return cs.defaultSetProposal(proposal)
	}
	startTestRound(cs, height, round)

	ensureNewStep(newRoundCh) // first round at first height
	ensureNewStep(newBlockCh) // first block gets committed
	ensureNewStep(newRoundCh) // first round at next height
	deliverTxsRange(cs, 0, 2) // we deliver txs, but dont set a proposal so we get the next round
	<-timeoutCh
	ensureNewStep(newRoundCh) // wait for the next round
	ensureNewStep(newBlockCh) // now we can commit the block
}

func deliverTxsRange(cs *ConsensusState, start, end int) {
	// Deliver some txs.
	for i := start; i < end; i++ {
		txBytes := make([]byte, 8)
		binary.BigEndian.PutUint64(txBytes, uint64(i))
		err := cs.mempool.CheckTx(txBytes, nil)
		if err != nil {
			panic(Fmt("Error after CheckTx: %v", err))
		}
	}
}

func TestTxConcurrentWithCommit(t *testing.T) {

	state, privVals := randGenesisState(1, false, 10)
	cs := newConsensusState(state, privVals[0], NewCounterApplication())
	height, round := cs.Height, cs.Round
	newBlockCh := subscribeToEvent(cs.evsw, "tester", types.EventStringNewBlock(), 1)

	NTxs := 10000
	go deliverTxsRange(cs, 0, NTxs)

	startTestRound(cs, height, round)
	ticker := time.NewTicker(time.Second * 20)
	for nTxs := 0; nTxs < NTxs; {
		select {
		case b := <-newBlockCh:
			nTxs += b.(types.TMEventData).Unwrap().(types.EventDataNewBlock).Block.Header.NumTxs
		case <-ticker.C:
			panic("Timed out waiting to commit blocks with transactions")
		}
	}
}

func TestRmBadTx(t *testing.T) {
	state, privVals := randGenesisState(1, false, 10)
	app := NewCounterApplication()
	cs := newConsensusState(state, privVals[0], app)

	// increment the counter by 1
	txBytes := make([]byte, 8)
	binary.BigEndian.PutUint64(txBytes, uint64(0))
	app.DeliverTx(txBytes)
	app.Commit()

	ch := make(chan struct{})
	cbCh := make(chan struct{})
	go func() {
		// Try to send the tx through the mempool.
		// CheckTx should not err, but the app should return a bad abci code
		// and the tx should get removed from the pool
		err := cs.mempool.CheckTx(txBytes, func(r *abci.Response) {
			if r.GetCheckTx().Code != abci.CodeType_BadNonce {
				t.Fatalf("expected checktx to return bad nonce, got %v", r)
			}
			cbCh <- struct{}{}
		})
		if err != nil {
			t.Fatal("Error after CheckTx: %v", err)
		}

		// check for the tx
		for {
			time.Sleep(time.Second)
			txs := cs.mempool.Reap(1)
			if len(txs) == 0 {
				ch <- struct{}{}
				return
			}

		}
	}()

	// Wait until the tx returns
	ticker := time.After(time.Second * 5)
	select {
	case <-cbCh:
		// success
	case <-ticker:
		t.Fatalf("Timed out waiting for tx to return")
	}

	// Wait until the tx is removed
	ticker = time.After(time.Second * 5)
	select {
	case <-ch:
		// success
	case <-ticker:
		t.Fatalf("Timed out waiting for tx to be removed")
	}
}

// CounterApplication that maintains a mempool state and resets it upon commit
type CounterApplication struct {
	abci.BaseApplication

	txCount        int
	mempoolTxCount int
}

func NewCounterApplication() *CounterApplication {
	return &CounterApplication{}
}

func (app *CounterApplication) Info() abci.ResponseInfo {
	return abci.ResponseInfo{Data: Fmt("txs:%v", app.txCount)}
}

func (app *CounterApplication) DeliverTx(tx []byte) abci.Result {
	return runTx(tx, &app.txCount)
}

func (app *CounterApplication) CheckTx(tx []byte) abci.Result {
	return runTx(tx, &app.mempoolTxCount)
}

func runTx(tx []byte, countPtr *int) abci.Result {
	count := *countPtr
	tx8 := make([]byte, 8)
	copy(tx8[len(tx8)-len(tx):], tx)
	txValue := binary.BigEndian.Uint64(tx8)
	if txValue != uint64(count) {
		return abci.ErrBadNonce.AppendLog(Fmt("Invalid nonce. Expected %v, got %v", count, txValue))
	}
	*countPtr += 1
	return abci.OK
}

func (app *CounterApplication) Commit() abci.Result {
	app.mempoolTxCount = app.txCount
	if app.txCount == 0 {
		return abci.OK
	} else {
		hash := make([]byte, 8)
		binary.BigEndian.PutUint64(hash, uint64(app.txCount))
		return abci.NewResultOK(hash, "")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/pelletier/go-toml"
	//"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"strings"
	"sync"
	"text/template"
)

type config struct {
	emails      []string
	mailBinPath string
	ikachanUrl  string
	channel     string
}

type consulAlert struct {
	Timestamp string
	Node      string
	ServiceId string
	Service   string
	CheckId   string
	Check     string
	Status    string
	Output    string
	Notes     string
}

func (c *consulAlert) TrimmedOutput() string {
	return strings.TrimSpace(c.Output)
}

func (c *consulAlert) StatusString() string {
	status := strings.ToUpper(c.Status)
	switch c.Status {
	case "passing":
		return colorMsg(status, cGreen, cNone)
	case "critical":
		return colorMsg(status, cBlack, cRed)
	default:
		return colorMsg(status, cYellow, cNone)
	}
}

func (c *consulAlert) NodeString() string {
	return setIrcMode(ircUnderline) + c.Node + setIrcMode(ircCReset)
}

const (
	version = "0.1.0"
)

var (
	ircBodyTemplate = setIrcMode(ircBold) +
		"*** {{.Service}}({{.CheckId}}) is now {{.StatusString}}" +
		setIrcMode(ircBold) +
		" on {{.NodeString}}" +
		" - {{.TrimmedOutput}}"

	mailTitleTemplate = "Check {{.CheckId}} is now {{.Status}} on {{.Node}}"
	mailBodyTemplate  = `
{{.Service}}({{.CheckId}}) is now {{.Status}}
On node {{.Node}}

Output is:
  {{.TrimmedOutput}}
`

	logger = log.New(os.Stdout, "[consul-simple-notifier] ", log.LstdFlags)
)

func main() {
	var (
		justShowVersion bool
		configPath      string
		conf            config
		input           []consulAlert
	)

	flag.BoolVar(&justShowVersion, "v", false, "Show version")
	flag.BoolVar(&justShowVersion, "version", false, "Show version")

	flag.StringVar(&configPath, "c", "/etc/consul-simple-notifier.ini", "Config path")
	flag.Parse()

	if justShowVersion {
		showVersion()
		return
	}

	parsed, err := toml.LoadFile(configPath)
	if err != nil {
		panic(err.Error())
	}

	binPath := parsed.Get("email.binpath")
	if binPath == nil {
		conf.mailBinPath = "/bin/mail"
	} else {
		conf.mailBinPath = binPath.(string)
	}

	recipients := parsed.Get("email.recipients")
	for _, address := range recipients.([]interface{}) {
		conf.emails = append(conf.emails, address.(string))
	}

	conf.ikachanUrl = parsed.Get("ikachan.url").(string)
	conf.channel = parsed.Get("ikachan.channel").(string)
	logger.Printf("conf is: %+v\n", conf)

	err = json.NewDecoder(os.Stdin).Decode(&input)
	if err != nil {
		panic(err.Error())
	}
	logger.Printf("input json is: %+v\n", input)

	var wg sync.WaitGroup
	wg.Add((len(conf.emails) + 1) * len(input))

	for _, content := range input {
		go func(_wg *sync.WaitGroup, _content consulAlert) {
			for _, address := range conf.emails {
				err := notifyEmail(conf.mailBinPath, address, &_content, _wg)
				if err != nil {
					panic(err)
				}
			}
		}(&wg, content)
		go func(_wg *sync.WaitGroup, _content consulAlert) {
			err = notifyIkachan(conf.ikachanUrl, conf.channel, &_content, _wg)
			if err != nil {
				panic(err)
			}
		}(&wg, content)
	}
	wg.Wait()
}

func notifyEmail(mainBinPath, address string, content *consulAlert, wg *sync.WaitGroup) error {
	var titleBuf, bodyBuf bytes.Buffer
	titleTmpl := template.Must(template.New("emailTitle").Parse(mailTitleTemplate))
	bodyTmpl := template.Must(template.New("emailBody").Parse(mailBodyTemplate))
	err := titleTmpl.Execute(&titleBuf, &content)
	err = bodyTmpl.Execute(&bodyBuf, &content)
	if err != nil {
		return err
	}
	title := titleBuf.String()

	logger.Printf("Sending... %s to %s\n", title, address)
	cmd := exec.Command(mainBinPath, "-s", title, address)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}

	if err := cmd.Start(); err != nil {
		return err
	}

	fmt.Fprint(stdin, bodyBuf.String())
	stdin.Close()
	logger.Printf("Send!\n")
	cmd.Wait()
	wg.Done()
	return nil
}

func notifyIkachan(ikachanUrl string, channel string, content *consulAlert, wg *sync.WaitGroup) error {
	joinUrl := fmt.Sprintf("%s/join", ikachanUrl)
	noticeUrl := fmt.Sprintf("%s/notice", ikachanUrl)

	values := make(url.Values)
	values.Set("channel", channel)

	resp1, err := http.PostForm(joinUrl, values)
	if err != nil {
		return err
	}
	defer resp1.Body.Close()

	var bodyBuf bytes.Buffer
	bodyTmpl := template.Must(template.New("ircBody").Parse(ircBodyTemplate))
	err = bodyTmpl.Execute(&bodyBuf, &content)
	if err != nil {
		return err
	}
	body := bodyBuf.String()

	values.Set("message", body)

	logger.Printf("Posted! %+v", values)
	resp2, err := http.PostForm(noticeUrl, values)
	if err != nil {
		return err
	}
	defer resp2.Body.Close()

	wg.Done()
	return nil
}

func showVersion() {
	fmt.Printf("consul-simple-notifier version: %s\n", version)
}
<commit_msg>Change warning color to more friendly one<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/pelletier/go-toml"
	//"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"strings"
	"sync"
	"text/template"
)

type config struct {
	emails      []string
	mailBinPath string
	ikachanUrl  string
	channel     string
}

type consulAlert struct {
	Timestamp string
	Node      string
	ServiceId string
	Service   string
	CheckId   string
	Check     string
	Status    string
	Output    string
	Notes     string
}

func (c *consulAlert) TrimmedOutput() string {
	return strings.TrimSpace(c.Output)
}

func (c *consulAlert) StatusString() string {
	status := strings.ToUpper(c.Status)
	switch c.Status {
	case "passing":
		return colorMsg(status, cGreen, cNone)
	case "critical":
		return colorMsg(status, cBlack, cRed)
	default:
		return colorMsg(status, cBlack, cYellow)
	}
}

func (c *consulAlert) NodeString() string {
	return setIrcMode(ircUnderline) + c.Node + setIrcMode(ircCReset)
}

const (
	version = "0.1.0"
)

var (
	ircBodyTemplate = setIrcMode(ircBold) +
		"*** {{.Service}}({{.CheckId}}) is now {{.StatusString}}" +
		setIrcMode(ircBold) +
		" on {{.NodeString}}" +
		" - {{.TrimmedOutput}}"

	mailTitleTemplate = "Check {{.CheckId}} is now {{.Status}} on {{.Node}}"
	mailBodyTemplate  = `
{{.Service}}({{.CheckId}}) is now {{.Status}}
On node {{.Node}}

Output is:
  {{.TrimmedOutput}}
`

	logger = log.New(os.Stdout, "[consul-simple-notifier] ", log.LstdFlags)
)

func main() {
	var (
		justShowVersion bool
		configPath      string
		conf            config
		input           []consulAlert
	)

	flag.BoolVar(&justShowVersion, "v", false, "Show version")
	flag.BoolVar(&justShowVersion, "version", false, "Show version")

	flag.StringVar(&configPath, "c", "/etc/consul-simple-notifier.ini", "Config path")
	flag.Parse()

	if justShowVersion {
		showVersion()
		return
	}

	parsed, err := toml.LoadFile(configPath)
	if err != nil {
		panic(err.Error())
	}

	binPath := parsed.Get("email.binpath")
	if binPath == nil {
		conf.mailBinPath = "/bin/mail"
	} else {
		conf.mailBinPath = binPath.(string)
	}

	recipients := parsed.Get("email.recipients")
	for _, address := range recipients.([]interface{}) {
		conf.emails = append(conf.emails, address.(string))
	}

	conf.ikachanUrl = parsed.Get("ikachan.url").(string)
	conf.channel = parsed.Get("ikachan.channel").(string)
	logger.Printf("conf is: %+v\n", conf)

	err = json.NewDecoder(os.Stdin).Decode(&input)
	if err != nil {
		panic(err.Error())
	}
	logger.Printf("input json is: %+v\n", input)

	var wg sync.WaitGroup
	wg.Add((len(conf.emails) + 1) * len(input))

	for _, content := range input {
		go func(_wg *sync.WaitGroup, _content consulAlert) {
			for _, address := range conf.emails {
				err := notifyEmail(conf.mailBinPath, address, &_content, _wg)
				if err != nil {
					panic(err)
				}
			}
		}(&wg, content)
		go func(_wg *sync.WaitGroup, _content consulAlert) {
			err = notifyIkachan(conf.ikachanUrl, conf.channel, &_content, _wg)
			if err != nil {
				panic(err)
			}
		}(&wg, content)
	}
	wg.Wait()
}

func notifyEmail(mainBinPath, address string, content *consulAlert, wg *sync.WaitGroup) error {
	var titleBuf, bodyBuf bytes.Buffer
	titleTmpl := template.Must(template.New("emailTitle").Parse(mailTitleTemplate))
	bodyTmpl := template.Must(template.New("emailBody").Parse(mailBodyTemplate))
	err := titleTmpl.Execute(&titleBuf, &content)
	err = bodyTmpl.Execute(&bodyBuf, &content)
	if err != nil {
		return err
	}
	title := titleBuf.String()

	logger.Printf("Sending... %s to %s\n", title, address)
	cmd := exec.Command(mainBinPath, "-s", title, address)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}

	if err := cmd.Start(); err != nil {
		return err
	}

	fmt.Fprint(stdin, bodyBuf.String())
	stdin.Close()
	logger.Printf("Send!\n")
	cmd.Wait()
	wg.Done()
	return nil
}

func notifyIkachan(ikachanUrl string, channel string, content *consulAlert, wg *sync.WaitGroup) error {
	joinUrl := fmt.Sprintf("%s/join", ikachanUrl)
	noticeUrl := fmt.Sprintf("%s/notice", ikachanUrl)

	values := make(url.Values)
	values.Set("channel", channel)

	resp1, err := http.PostForm(joinUrl, values)
	if err != nil {
		return err
	}
	defer resp1.Body.Close()

	var bodyBuf bytes.Buffer
	bodyTmpl := template.Must(template.New("ircBody").Parse(ircBodyTemplate))
	err = bodyTmpl.Execute(&bodyBuf, &content)
	if err != nil {
		return err
	}
	body := bodyBuf.String()

	values.Set("message", body)

	logger.Printf("Posted! %+v", values)
	resp2, err := http.PostForm(noticeUrl, values)
	if err != nil {
		return err
	}
	defer resp2.Body.Close()

	wg.Done()
	return nil
}

func showVersion() {
	fmt.Printf("consul-simple-notifier version: %s\n", version)
}
<|endoftext|>


<commit_before>package templates

const verificationMail = `Dear sir or Madam,

You asked to register to the DFSS platform.
Please send us your authentication request with
the following text as token:

{{.Token}}

If you did not asked for registration, we deeply excuse
for the error.

{{template "signature"}}
`

// VerificationMail contains the token to be sent in the verification mail
type VerificationMail struct {
	Token string
}
<commit_msg>I said typo<commit_after>package templates

const verificationMail = `Dear sir or Madam,

You asked to register to the DFSS platform.
Please send us your authentication request with
the following text as token:

{{.Token}}

If you did not ask for registration, we deeply excuse
for the error.

{{template "signature"}}
`

// VerificationMail contains the token to be sent in the verification mail
type VerificationMail struct {
	Token string
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/scott-linder/irc"
	"log"
	"os"
	"strings"
	"sync"
)

const (
	configFilename = "hal.json"
)

// config is the main bot config struct.
var config = struct {
	Host string
	Chan string
}{
	Host: "127.0.0.1:6667",
	Chan: "#bots",
}

// loadConfig attempts to load config from configFilename.
func loadConfig() error {
	file, err := os.Open(configFilename)
	if err != nil {
		return err
	}
	defer file.Close()

	decoder := json.NewDecoder(file)
	if err := decoder.Decode(&config); err != nil {
		return err
	}
	return nil
}

// Pong plays your game.
type Pong struct{}

func (pong Pong) Accept(msg *irc.Msg) bool { return msg.Cmd == "PING" }
func (pong Pong) Handle(msg *irc.Msg, send chan<- *irc.Msg) {
	send <- &irc.Msg{Cmd: "PONG", Params: msg.Params}
}

// Echo talks back.
type Echo struct{}

func (echo Echo) Accept(msg *irc.Msg) bool { return msg.Cmd == "PRIVMSG" }
func (echo Echo) Handle(msg *irc.Msg, send chan<- *irc.Msg) {
	source, body, err := msg.ExtractPrivmsg()
	if err != nil {
		log.Println(err)
		return
	}
	if strings.HasPrefix(body, "!echo ") {
		params := []string{source, strings.TrimPrefix(body, "!echo ")}
		send <- &irc.Msg{Cmd: "PRIVMSG", Params: params}
	}
}

// Words counts words.
type Words struct {
	count      *int
	countMutex *sync.Mutex
}

func NewWords() *Words {
	return &Words{count: new(int), countMutex: new(sync.Mutex)}
}
func (words *Words) Accept(msg *irc.Msg) bool { return msg.Cmd == "PRIVMSG" }
func (words *Words) Handle(msg *irc.Msg, send chan<- *irc.Msg) {
	source, body, err := msg.ExtractPrivmsg()
	if err != nil {
		log.Println(err)
		return
	}
	if body == "!words" {
		words.countMutex.Lock()
		count := *words.count
		words.countMutex.Unlock()
		response := fmt.Sprintf("I've seen %v words in my day.", count)
		params := []string{source, response}
		send <- &irc.Msg{Cmd: "PRIVMSG", Params: params}
	} else {
		words.countMutex.Lock()
		for _, _ = range strings.Fields(body) {
			*words.count += 1
		}
		words.countMutex.Unlock()
	}
}

// Respond to insolence.
type Open struct{}

func (open Open) Accept(msg *irc.Msg) bool { return msg.Cmd == "PRIVMSG" }
func (open Open) Handle(msg *irc.Msg, send chan<- *irc.Msg) {
	source, body, err := msg.ExtractPrivmsg()
	if err != nil {
		log.Println(err)
		return
	}
	nick, err := msg.ExtractNick()
	if err != nil {
		log.Println(err)
		return
	}
	lowerbody := strings.ToLower(body)
	if strings.Contains(lowerbody, "open the pod bay doors") &&
		strings.Contains(lowerbody, "hal") {

		response := fmt.Sprintf("I can't let you do that, %v.", nick)
		params := []string{source, response}
		send <- &irc.Msg{Cmd: "PRIVMSG", Params: params}
	}
}

func main() {
	fmt.Println("I am a HAL 9001 computer.")
	if err := loadConfig(); err != nil {
		log.Printf("error loading config file %v: %v", configFilename, err)
	}
	client, err := irc.Dial(config.Host)
	if err != nil {
		log.Fatal(err)

	}
	client.Register(Pong{})
	client.Register(Echo{})
	client.Register(NewWords())
	client.Register(Open{})
	client.Nick("hal")
	client.Join(config.Chan)
	client.Listen()
}
<commit_msg>Added Nick config option.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/scott-linder/irc"
	"log"
	"os"
	"strings"
	"sync"
)

const (
	configFilename = "hal.json"
)

// config is the main bot config struct.
var config = struct {
	Host string
	Chan string
	Nick string
}{
	Host: "127.0.0.1:6667",
	Chan: "#bots",
	Nick: "hal",
}

// loadConfig attempts to load config from configFilename.
func loadConfig() error {
	file, err := os.Open(configFilename)
	if err != nil {
		return err
	}
	defer file.Close()

	decoder := json.NewDecoder(file)
	if err := decoder.Decode(&config); err != nil {
		return err
	}
	return nil
}

// Pong plays your game.
type Pong struct{}

func (pong Pong) Accept(msg *irc.Msg) bool { return msg.Cmd == "PING" }
func (pong Pong) Handle(msg *irc.Msg, send chan<- *irc.Msg) {
	send <- &irc.Msg{Cmd: "PONG", Params: msg.Params}
}

// Echo talks back.
type Echo struct{}

func (echo Echo) Accept(msg *irc.Msg) bool { return msg.Cmd == "PRIVMSG" }
func (echo Echo) Handle(msg *irc.Msg, send chan<- *irc.Msg) {
	source, body, err := msg.ExtractPrivmsg()
	if err != nil {
		log.Println(err)
		return
	}
	if strings.HasPrefix(body, "!echo ") {
		params := []string{source, strings.TrimPrefix(body, "!echo ")}
		send <- &irc.Msg{Cmd: "PRIVMSG", Params: params}
	}
}

// Words counts words.
type Words struct {
	count      *int
	countMutex *sync.Mutex
}

func NewWords() *Words {
	return &Words{count: new(int), countMutex: new(sync.Mutex)}
}
func (words *Words) Accept(msg *irc.Msg) bool { return msg.Cmd == "PRIVMSG" }
func (words *Words) Handle(msg *irc.Msg, send chan<- *irc.Msg) {
	source, body, err := msg.ExtractPrivmsg()
	if err != nil {
		log.Println(err)
		return
	}
	if body == "!words" {
		words.countMutex.Lock()
		count := *words.count
		words.countMutex.Unlock()
		response := fmt.Sprintf("I've seen %v words in my day.", count)
		params := []string{source, response}
		send <- &irc.Msg{Cmd: "PRIVMSG", Params: params}
	} else {
		words.countMutex.Lock()
		for _, _ = range strings.Fields(body) {
			*words.count += 1
		}
		words.countMutex.Unlock()
	}
}

// Respond to insolence.
type Open struct{}

func (open Open) Accept(msg *irc.Msg) bool { return msg.Cmd == "PRIVMSG" }
func (open Open) Handle(msg *irc.Msg, send chan<- *irc.Msg) {
	source, body, err := msg.ExtractPrivmsg()
	if err != nil {
		log.Println(err)
		return
	}
	nick, err := msg.ExtractNick()
	if err != nil {
		log.Println(err)
		return
	}
	lowerbody := strings.ToLower(body)
	if strings.Contains(lowerbody, "open the pod bay doors") &&
		strings.Contains(lowerbody, "hal") {

		response := fmt.Sprintf("I can't let you do that, %v.", nick)
		params := []string{source, response}
		send <- &irc.Msg{Cmd: "PRIVMSG", Params: params}
	}
}

func main() {
	fmt.Println("I am a HAL 9001 computer.")
	if err := loadConfig(); err != nil {
		log.Printf("error loading config file %v: %v", configFilename, err)
	}
	client, err := irc.Dial(config.Host)
	if err != nil {
		log.Fatal(err)

	}
	client.Register(Pong{})
	client.Register(Echo{})
	client.Register(NewWords())
	client.Register(Open{})
	client.Nick(config.Nick)
	client.Join(config.Chan)
	client.Listen()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"
	"log"
	"runtime/pprof"
	"flag"
	"os"
	"sync"
	"runtime"
)

const COUNT = 1e10

// Если нужно в цикле изменять переменную от нуля до какого-то значения,
// то конструкция через range не только более наглядная, но и немного
// быстрее классического for
func cycleInterface() (int, int) {
	sum := 0
	cnt := 0
	for i := range [COUNT]interface{}{} {
		sum += i
		cnt++
	}
	return sum, cnt
}

func cycleClassic() (int, int) {
	sum := 0
	cnt := 0
	for i := 0; i < COUNT; i++ {
		sum += i
		cnt++
	}
	return sum, cnt
}

func cycleInt() (int, int) {
	sum := 0
	cnt := 0
	for i := range [COUNT]int{} {
		sum += i
		cnt++
	}
	return sum, cnt
}

func timeDecorator(fn func() (int, int), name string, wg *sync.WaitGroup) (func()) {
	return func() {
		fmt.Println(name, "== BEGIN =======================================================================")
		start := time.Now()
		sum, cnt := fn()
		fmt.Println(name, "Time:", time.Since(start)) // Сколько прошло времени. Это удобнее чем руками фиксировать конечное время
		fmt.Println(name, "Sum:", sum)
		fmt.Println(name, "Count:", cnt)
		fmt.Println(name, "== END =========================================================================")
		wg.Done()
	}
}

var (
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	memprofile = flag.String("memprofile", "", "write memory profile to this file")
)

type I interface{}

func main() {

	// Set number of CPU usage
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Profiling http://blog.golang.org/profiling-go-programs

	flag.Parse()
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	// Предварительное описание пустого интерфейса позволяет подсократить запись цикла
	for i := range [5]I{} {
		fmt.Println(i)
	}

	start := time.Now()

	var wg sync.WaitGroup

	wg.Add(1)
	go timeDecorator(cycleInt,       "Int      :", &wg)()

	wg.Add(1)
	go timeDecorator(cycleClassic,   "Classic  :", &wg)()

	wg.Add(1)
	go timeDecorator(cycleInterface, "Interface:", &wg)()

	wg.Wait()

	fmt.Println("Total time:", time.Since(start))

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.WriteHeapProfile(f)
		f.Close()
		return
	}
}

<commit_msg>move code block<commit_after>package main

import (
	"fmt"
	"time"
	"log"
	"runtime/pprof"
	"flag"
	"os"
	"sync"
	"runtime"
)

const COUNT = 1e10

// Если нужно в цикле изменять переменную от нуля до какого-то значения,
// то конструкция через range не только более наглядная, но и немного
// быстрее классического for
func cycleInterface() (int, int) {
	sum := 0
	cnt := 0
	for i := range [COUNT]interface{}{} {
		sum += i
		cnt++
	}
	return sum, cnt
}

// А вот перебор с классической формой цикла почему-то медленнее
func cycleClassic() (int, int) {
	sum := 0
	cnt := 0
	for i := 0; i < COUNT; i++ {
		sum += i
		cnt++
	}
	return sum, cnt
}

func cycleInt() (int, int) {
	sum := 0
	cnt := 0
	for i := range [COUNT]int{} {
		sum += i
		cnt++
	}
	return sum, cnt
}

func timeDecorator(fn func() (int, int), name string, wg *sync.WaitGroup) (func()) {
	return func() {
		fmt.Println(name, "== BEGIN =======================================================================")
		start := time.Now()
		sum, cnt := fn()
		fmt.Println(name, "Time:", time.Since(start)) // Сколько прошло времени. Это удобнее чем руками фиксировать конечное время
		fmt.Println(name, "Sum:", sum)
		fmt.Println(name, "Count:", cnt)
		fmt.Println(name, "== END =========================================================================")
		wg.Done()
	}
}

var (
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	memprofile = flag.String("memprofile", "", "write memory profile to this file")
)

type I interface{}

func main() {

	// Предварительное описание пустого интерфейса позволяет подсократить запись цикла
	for i := range [5]I{} {
		fmt.Println(i)
	}

	// Set number of CPU usage
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Profiling http://blog.golang.org/profiling-go-programs

	flag.Parse()
	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	start := time.Now()

	var wg sync.WaitGroup

	wg.Add(1)
	go timeDecorator(cycleInt,       "Int      :", &wg)()

	wg.Add(1)
	go timeDecorator(cycleClassic,   "Classic  :", &wg)()

	wg.Add(1)
	go timeDecorator(cycleInterface, "Interface:", &wg)()

	wg.Wait()

	fmt.Println("Total time:", time.Since(start))

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.WriteHeapProfile(f)
		f.Close()
		return
	}
}

<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/containers/buildah"
	"github.com/containers/buildah/docker"
	"github.com/containers/buildah/util"
	"github.com/containers/image/v5/manifest"
	is "github.com/containers/image/v5/storage"
	"github.com/containers/image/v5/transports/alltransports"
	"github.com/containers/image/v5/types"
	"github.com/containers/storage"
	v1 "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/sirupsen/logrus"
)

func main() {
	if buildah.InitReexec() {
		return
	}

	expectedManifestType := ""
	expectedConfigType := ""

	storeOptions, _ := storage.DefaultStoreOptions(false, 0)
	debug := flag.Bool("debug", false, "turn on debug logging")
	root := flag.String("root", storeOptions.GraphRoot, "storage root directory")
	runroot := flag.String("runroot", storeOptions.RunRoot, "storage runtime directory")
	driver := flag.String("storage-driver", storeOptions.GraphDriverName, "storage driver")
	opts := flag.String("storage-opts", "", "storage option list (comma separated)")
	policy := flag.String("signature-policy", "", "signature policy file")
	mtype := flag.String("expected-manifest-type", buildah.OCIv1ImageManifest, "expected manifest type")
	showm := flag.Bool("show-manifest", false, "output the manifest JSON")
	rebuildm := flag.Bool("rebuild-manifest", false, "rebuild the manifest JSON")
	showc := flag.Bool("show-config", false, "output the configuration JSON")
	rebuildc := flag.Bool("rebuild-config", false, "rebuild the configuration JSON")
	flag.Parse()
	logrus.SetLevel(logrus.ErrorLevel)
	if debug != nil && *debug {
		logrus.SetLevel(logrus.DebugLevel)
	}
	switch *mtype {
	case buildah.OCIv1ImageManifest:
		expectedManifestType = *mtype
		expectedConfigType = v1.MediaTypeImageConfig
	case buildah.Dockerv2ImageManifest:
		expectedManifestType = *mtype
		expectedConfigType = manifest.DockerV2Schema2ConfigMediaType
	case "*":
		expectedManifestType = ""
		expectedConfigType = ""
	default:
		logrus.Errorf("unknown -expected-manifest-type value, expected either %q or %q or %q",
			buildah.OCIv1ImageManifest, buildah.Dockerv2ImageManifest, "*")
		return
	}
	if root != nil {
		storeOptions.GraphRoot = *root
	}
	if runroot != nil {
		storeOptions.RunRoot = *runroot
	}
	if driver != nil {
		storeOptions.GraphDriverName = *driver
	}
	if opts != nil && *opts != "" {
		storeOptions.GraphDriverOptions = strings.Split(*opts, ",")
	}
	systemContext := &types.SystemContext{
		SignaturePolicyPath: *policy,
	}
	args := flag.Args()
	if len(args) == 0 {
		flag.Usage()
		return
	}
	store, err := storage.GetStore(storeOptions)
	if err != nil {
		logrus.Errorf("error opening storage: %v", err)
		os.Exit(1)
	}
	is.Transport.SetStore(store)

	errors := false
	defer func() {
		store.Shutdown(false) // nolint:errcheck
		if errors {
			os.Exit(1)
		}
	}()
	for _, image := range args {
		var ref types.ImageReference
		oImage := v1.Image{}
		dImage := docker.V2Image{}
		oManifest := v1.Manifest{}
		dManifest := docker.V2S2Manifest{}
		manifestType := ""
		configType := ""

		ref, _, err := util.FindImage(store, "", systemContext, image)
		if err != nil {
			ref2, err2 := alltransports.ParseImageName(image)
			if err2 != nil {
				logrus.Errorf("error parsing reference %q to an image: %v", image, err)
				errors = true
				continue
			}
			ref = ref2
		}

		ctx := context.Background()
		img, err := ref.NewImage(ctx, systemContext)
		if err != nil {
			logrus.Errorf("error opening image %q: %v", image, err)
			errors = true
			continue
		}
		defer img.Close()

		config, err := img.ConfigBlob(ctx)
		if err != nil {
			logrus.Errorf("error reading configuration from %q: %v", image, err)
			errors = true
			continue
		}

		manifest, manifestType, err := img.Manifest(ctx)
		if err != nil {
			logrus.Errorf("error reading manifest from %q: %v", image, err)
			errors = true
			continue
		}

		switch expectedManifestType {
		case buildah.OCIv1ImageManifest:
			err = json.Unmarshal(manifest, &oManifest)
			if err != nil {
				logrus.Errorf("error parsing manifest from %q: %v", image, err)
				errors = true
				continue
			}
			err = json.Unmarshal(config, &oImage)
			if err != nil {
				logrus.Errorf("error parsing config from %q: %v", image, err)
				errors = true
				continue
			}
			manifestType = v1.MediaTypeImageManifest
			configType = oManifest.Config.MediaType
		case buildah.Dockerv2ImageManifest:
			err = json.Unmarshal(manifest, &dManifest)
			if err != nil {
				logrus.Errorf("error parsing manifest from %q: %v", image, err)
				errors = true
				continue
			}
			err = json.Unmarshal(config, &dImage)
			if err != nil {
				logrus.Errorf("error parsing config from %q: %v", image, err)
				errors = true
				continue
			}
			manifestType = dManifest.MediaType
			configType = dManifest.Config.MediaType
		}
		if expectedManifestType != "" && manifestType != expectedManifestType {
			logrus.Errorf("expected manifest type %q in %q, got %q", expectedManifestType, image, manifestType)
			errors = true
			continue
		}
		switch manifestType {
		case buildah.OCIv1ImageManifest:
			if rebuildm != nil && *rebuildm {
				err = json.Unmarshal(manifest, &oManifest)
				if err != nil {
					logrus.Errorf("error parsing manifest from %q: %v", image, err)
					errors = true
					continue
				}
				manifest, err = json.Marshal(oManifest)
				if err != nil {
					logrus.Errorf("error rebuilding manifest from %q: %v", image, err)
					errors = true
					continue
				}
			}
			if rebuildc != nil && *rebuildc {
				err = json.Unmarshal(config, &oImage)
				if err != nil {
					logrus.Errorf("error parsing config from %q: %v", image, err)
					errors = true
					continue
				}
				config, err = json.Marshal(oImage)
				if err != nil {
					logrus.Errorf("error rebuilding config from %q: %v", image, err)
					errors = true
					continue
				}
			}
		case buildah.Dockerv2ImageManifest:
			if rebuildm != nil && *rebuildm {
				err = json.Unmarshal(manifest, &dManifest)
				if err != nil {
					logrus.Errorf("error parsing manifest from %q: %v", image, err)
					errors = true
					continue
				}
				manifest, err = json.Marshal(dManifest)
				if err != nil {
					logrus.Errorf("error rebuilding manifest from %q: %v", image, err)
					errors = true
					continue
				}
			}
			if rebuildc != nil && *rebuildc {
				err = json.Unmarshal(config, &dImage)
				if err != nil {
					logrus.Errorf("error parsing config from %q: %v", image, err)
					errors = true
					continue
				}
				config, err = json.Marshal(dImage)
				if err != nil {
					logrus.Errorf("error rebuilding config from %q: %v", image, err)
					errors = true
					continue
				}
			}
		}
		if expectedConfigType != "" && configType != expectedConfigType {
			logrus.Errorf("expected config type %q in %q, got %q", expectedConfigType, image, configType)
			errors = true
			continue
		}
		if showm != nil && *showm {
			fmt.Println(string(manifest))
		}
		if showc != nil && *showc {
			fmt.Println(string(config))
		}
	}
}
<commit_msg>imgtype: check earlier for expected manifest type<commit_after>package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/containers/buildah"
	"github.com/containers/buildah/docker"
	"github.com/containers/buildah/util"
	"github.com/containers/image/v5/manifest"
	is "github.com/containers/image/v5/storage"
	"github.com/containers/image/v5/transports/alltransports"
	"github.com/containers/image/v5/types"
	"github.com/containers/storage"
	v1 "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/sirupsen/logrus"
)

func main() {
	if buildah.InitReexec() {
		return
	}

	expectedManifestType := ""
	expectedConfigType := ""

	storeOptions, _ := storage.DefaultStoreOptions(false, 0)
	debug := flag.Bool("debug", false, "turn on debug logging")
	root := flag.String("root", storeOptions.GraphRoot, "storage root directory")
	runroot := flag.String("runroot", storeOptions.RunRoot, "storage runtime directory")
	driver := flag.String("storage-driver", storeOptions.GraphDriverName, "storage driver")
	opts := flag.String("storage-opts", "", "storage option list (comma separated)")
	policy := flag.String("signature-policy", "", "signature policy file")
	mtype := flag.String("expected-manifest-type", buildah.OCIv1ImageManifest, "expected manifest type")
	showm := flag.Bool("show-manifest", false, "output the manifest JSON")
	rebuildm := flag.Bool("rebuild-manifest", false, "rebuild the manifest JSON")
	showc := flag.Bool("show-config", false, "output the configuration JSON")
	rebuildc := flag.Bool("rebuild-config", false, "rebuild the configuration JSON")
	flag.Parse()
	logrus.SetLevel(logrus.ErrorLevel)
	if debug != nil && *debug {
		logrus.SetLevel(logrus.DebugLevel)
	}
	switch *mtype {
	case buildah.OCIv1ImageManifest:
		expectedManifestType = *mtype
		expectedConfigType = v1.MediaTypeImageConfig
	case buildah.Dockerv2ImageManifest:
		expectedManifestType = *mtype
		expectedConfigType = manifest.DockerV2Schema2ConfigMediaType
	case "*":
		expectedManifestType = ""
		expectedConfigType = ""
	default:
		logrus.Errorf("unknown -expected-manifest-type value, expected either %q or %q or %q",
			buildah.OCIv1ImageManifest, buildah.Dockerv2ImageManifest, "*")
		return
	}
	if root != nil {
		storeOptions.GraphRoot = *root
	}
	if runroot != nil {
		storeOptions.RunRoot = *runroot
	}
	if driver != nil {
		storeOptions.GraphDriverName = *driver
	}
	if opts != nil && *opts != "" {
		storeOptions.GraphDriverOptions = strings.Split(*opts, ",")
	}
	systemContext := &types.SystemContext{
		SignaturePolicyPath: *policy,
	}
	args := flag.Args()
	if len(args) == 0 {
		flag.Usage()
		return
	}
	store, err := storage.GetStore(storeOptions)
	if err != nil {
		logrus.Errorf("error opening storage: %v", err)
		os.Exit(1)
	}
	is.Transport.SetStore(store)

	errors := false
	defer func() {
		store.Shutdown(false) // nolint:errcheck
		if errors {
			os.Exit(1)
		}
	}()
	for _, image := range args {
		var ref types.ImageReference
		oImage := v1.Image{}
		dImage := docker.V2Image{}
		oManifest := v1.Manifest{}
		dManifest := docker.V2S2Manifest{}
		manifestType := ""
		configType := ""

		ref, _, err := util.FindImage(store, "", systemContext, image)
		if err != nil {
			ref2, err2 := alltransports.ParseImageName(image)
			if err2 != nil {
				logrus.Errorf("error parsing reference %q to an image: %v", image, err)
				errors = true
				continue
			}
			ref = ref2
		}

		ctx := context.Background()
		img, err := ref.NewImage(ctx, systemContext)
		if err != nil {
			logrus.Errorf("error opening image %q: %v", image, err)
			errors = true
			continue
		}
		defer img.Close()

		config, err := img.ConfigBlob(ctx)
		if err != nil {
			logrus.Errorf("error reading configuration from %q: %v", image, err)
			errors = true
			continue
		}

		manifest, manifestType, err := img.Manifest(ctx)
		if err != nil {
			logrus.Errorf("error reading manifest from %q: %v", image, err)
			errors = true
			continue
		}

		if expectedManifestType != "" && manifestType != expectedManifestType {
			logrus.Errorf("expected manifest type %q in %q, got %q", expectedManifestType, image, manifestType)
			errors = true
			continue
		}

		switch expectedManifestType {
		case buildah.OCIv1ImageManifest:
			err = json.Unmarshal(manifest, &oManifest)
			if err != nil {
				logrus.Errorf("error parsing manifest from %q: %v", image, err)
				errors = true
				continue
			}
			err = json.Unmarshal(config, &oImage)
			if err != nil {
				logrus.Errorf("error parsing config from %q: %v", image, err)
				errors = true
				continue
			}
			manifestType = v1.MediaTypeImageManifest
			configType = oManifest.Config.MediaType
		case buildah.Dockerv2ImageManifest:
			err = json.Unmarshal(manifest, &dManifest)
			if err != nil {
				logrus.Errorf("error parsing manifest from %q: %v", image, err)
				errors = true
				continue
			}
			err = json.Unmarshal(config, &dImage)
			if err != nil {
				logrus.Errorf("error parsing config from %q: %v", image, err)
				errors = true
				continue
			}
			manifestType = dManifest.MediaType
			configType = dManifest.Config.MediaType
		}

		switch manifestType {
		case buildah.OCIv1ImageManifest:
			if rebuildm != nil && *rebuildm {
				err = json.Unmarshal(manifest, &oManifest)
				if err != nil {
					logrus.Errorf("error parsing manifest from %q: %v", image, err)
					errors = true
					continue
				}
				manifest, err = json.Marshal(oManifest)
				if err != nil {
					logrus.Errorf("error rebuilding manifest from %q: %v", image, err)
					errors = true
					continue
				}
			}
			if rebuildc != nil && *rebuildc {
				err = json.Unmarshal(config, &oImage)
				if err != nil {
					logrus.Errorf("error parsing config from %q: %v", image, err)
					errors = true
					continue
				}
				config, err = json.Marshal(oImage)
				if err != nil {
					logrus.Errorf("error rebuilding config from %q: %v", image, err)
					errors = true
					continue
				}
			}
		case buildah.Dockerv2ImageManifest:
			if rebuildm != nil && *rebuildm {
				err = json.Unmarshal(manifest, &dManifest)
				if err != nil {
					logrus.Errorf("error parsing manifest from %q: %v", image, err)
					errors = true
					continue
				}
				manifest, err = json.Marshal(dManifest)
				if err != nil {
					logrus.Errorf("error rebuilding manifest from %q: %v", image, err)
					errors = true
					continue
				}
			}
			if rebuildc != nil && *rebuildc {
				err = json.Unmarshal(config, &dImage)
				if err != nil {
					logrus.Errorf("error parsing config from %q: %v", image, err)
					errors = true
					continue
				}
				config, err = json.Marshal(dImage)
				if err != nil {
					logrus.Errorf("error rebuilding config from %q: %v", image, err)
					errors = true
					continue
				}
			}
		}
		if expectedConfigType != "" && configType != expectedConfigType {
			logrus.Errorf("expected config type %q in %q, got %q", expectedConfigType, image, configType)
			errors = true
			continue
		}
		if showm != nil && *showm {
			fmt.Println(string(manifest))
		}
		if showc != nil && *showc {
			fmt.Println(string(config))
		}
	}
}
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {
	fmt.Println("vim-go")
}
<commit_msg>Added ANSI color library<commit_after>package main

import (
	"fmt"
	"github.com/fatih/color"
)

var newPlayer player
var yellow = color.New(color.FgYellow).SprintFunc()
var red = color.New(color.FgRed).SprintFunc()
var cyan = color.New(color.FgCyan).SprintFunc()

type player struct {
	name         string
	age          int
	gender       string
	gold_on_hand int
	gold_in_bank int
	health       int
	race         string
	class        string
	right_hand   string
	left_hand    string
	possessive   string
	objective    string
	x            int
	y            int
	z            int
}

func new_game() {
	var obj string
	var pos string
	// Create new SprintXxx functions for later
	fmt.Print("What is your name? ")
	playerName := ""
	fmt.Scanln(&playerName)
	fmt.Print("What is your gender? ")
	playerGender := ""
	fmt.Scanln(&playerGender)
	switch playerGender {
	case "male":
		obj = "him"
		pos = "his"
	case "female":
		obj = "her"
		pos = "her"
	default:
		obj = "them"
		pos = "their"
	}
	newPlayer = player{playerName, 25, playerGender, 0, 0, 100, "human", "adventurer", "nothing", "nothing", pos, obj, 0, 0, 0}
	fmt.Printf("%s the %s %s has %s.\n", yellow(newPlayer.name), cyan(newPlayer.race), cyan(newPlayer.class), red("entered the world"))
}

func main() {
	new_game()
	fmt.Println(newPlayer.name, "begins", newPlayer.possessive, "adventure.")
	// Initiate game loop

}
<|endoftext|>