<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"fmt"
	"time"

	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	jsonw "github.com/keybase/go-jsonw"
	"golang.org/x/net/context"
	"runtime/debug"
	"stathat.com/c/ramcache"
)

type ResolveResult struct {
	uid                keybase1.UID
	body               *jsonw.Wrapper
	err                error
	queriedKbUsername  string
	queriedByUID       bool
	resolvedKbUsername string
	cachedAt           time.Time
	mutable            bool
}

func (res ResolveResult) String() string {
	return fmt.Sprintf("{uid:%s err:%s mutable:%v}", res.uid, ErrToOk(res.err), res.mutable)
}

const (
	resolveCacheTTL           = 12 * time.Hour
	ResolveCacheMaxAge        = 12 * time.Hour
	ResolveCacheMaxAgeMutable = 20 * time.Minute
	resolveCacheMaxAgeErrored = 5 * time.Second
)

func (res *ResolveResult) GetUID() keybase1.UID {
	return res.uid
}

func (res *ResolveResult) GetUsername() string {
	return res.resolvedKbUsername
}
func (res *ResolveResult) GetNormalizedUsername() NormalizedUsername {
	return NewNormalizedUsername(res.GetUsername())
}
func (res *ResolveResult) GetNormalizedQueriedUsername() NormalizedUsername {
	return NewNormalizedUsername(res.queriedKbUsername)
}

func (res *ResolveResult) WasKBAssertion() bool {
	return res.queriedKbUsername != "" || res.queriedByUID
}

func (res *ResolveResult) GetError() error {
	return res.err
}

func (res *ResolveResult) GetBody() *jsonw.Wrapper {
	return res.body
}

func (r *Resolver) ResolveWithBody(input string) ResolveResult {
	return r.resolve(input, true)
}

func (r *Resolver) Resolve(input string) ResolveResult {
	return r.resolve(input, false)
}

func (r *Resolver) resolve(input string, withBody bool) (res ResolveResult) {
	defer r.G().Trace(fmt.Sprintf("Resolving username %q", input), func() error { return res.err })()

	var au AssertionURL
	if au, res.err = ParseAssertionURL(r.G().MakeAssertionContext(), input, false); res.err != nil {
		return res
	}
	res = r.resolveURL(context.TODO(), au, input, withBody, false)
	return res
}

func (r *Resolver) ResolveFullExpression(ctx context.Context, input string) (res ResolveResult) {
	return r.resolveFullExpression(ctx, input, false, false)
}

func (r *Resolver) ResolveFullExpressionNeedUsername(ctx context.Context, input string) (res ResolveResult) {
	return r.resolveFullExpression(ctx, input, false, true)
}

func (r *Resolver) ResolveFullExpressionWithBody(ctx context.Context, input string) (res ResolveResult) {
	return r.resolveFullExpression(ctx, input, true, false)
}

func (r *Resolver) resolveFullExpression(ctx context.Context, input string, withBody bool, needUsername bool) (res ResolveResult) {
	defer r.G().CVTrace(ctx, VLog1, fmt.Sprintf("Resolver#resolveFullExpression(%q)", input), func() error { return res.err })()

	var expr AssertionExpression
	expr, res.err = AssertionParseAndOnly(r.G().MakeAssertionContext(), input)
	if res.err != nil {
		return res
	}
	u := FindBestIdentifyComponentURL(expr)
	if u == nil {
		res.err = ResolutionError{Input: input, Msg: "Cannot find a resolvable factor"}
		return res
	}
	return r.resolveURL(ctx, u, input, withBody, needUsername)
}

func (res *ResolveResult) addKeybaseNameIfKnown(au AssertionURL) {
	if au.IsKeybase() && len(res.resolvedKbUsername) == 0 {
		res.resolvedKbUsername = au.GetValue()
	}
}

func (r *Resolver) getFromDiskCache(ctx context.Context, key string, au AssertionURL) (ret *ResolveResult) {
	defer r.G().CVTraceOK(ctx, VLog1, fmt.Sprintf("Resolver#getFromDiskCache(%q)", key), func() bool { return ret != nil })()
	var uid keybase1.UID
	found, err := r.G().LocalDb.GetInto(&uid, resolveDbKey(key))
	r.Stats.diskGets++
	if err != nil {
		r.G().Log.CWarningf(ctx, "Problem fetching resolve result from local DB: %s", err)
		return nil
	}
	if !found {
		r.Stats.diskGetMisses++
		return nil
	}
	if uid.IsNil() {
		r.G().Log.CWarningf(ctx, "nil UID found in disk cache")
		return nil
	}
	r.Stats.diskGetHits++
	return &ResolveResult{uid: uid}
}

func isMutable(au AssertionURL) bool {
	return !(au.IsUID() || au.IsKeybase())
}

func (r *Resolver) resolveURL(ctx context.Context, au AssertionURL, input string, withBody bool, needUsername bool) (res ResolveResult) {
	ck := au.CacheKey()

	// Debug succintly what happened in the resolution
	var trace string
	defer func() {
		r.G().Log.CDebugf(ctx, "| Resolver#resolveURL(%s) -> %s [trace:%s]", ck, res, trace)
	}()

	// A standard keybase UID, so it's already resolved... unless we explicitly
	// need it!
	if !needUsername {
		if tmp := au.ToUID(); tmp.Exists() {
			trace += "u"
			return ResolveResult{uid: tmp}
		}
	}

	if p := r.getFromMemCache(ctx, ck, au); p != nil && (!needUsername || len(p.resolvedKbUsername) > 0) {
		trace += "m"
		return *p
	}

	if p := r.getFromDiskCache(ctx, ck, au); p != nil && (!needUsername || len(p.resolvedKbUsername) > 0) {
		p.mutable = isMutable(au)
		r.putToMemCache(ck, *p)
		trace += "d"
		return *p
	}

	trace += "s"
	res = r.resolveURLViaServerLookup(ctx, au, input, withBody)

	// Cache for a shorter period of time if it's not a Keybase identity
	res.mutable = isMutable(au)
	r.putToMemCache(ck, res)

	// We only put to disk cache if it's a Keybase-type assertion. In particular, UIDs
	// are **not** stored to disk.
	if au.IsKeybase() {
		trace += "p"
		r.putToDiskCache(ctx, ck, res)
	}

	return res
}

func (r *Resolver) resolveURLViaServerLookup(ctx context.Context, au AssertionURL, input string, withBody bool) (res ResolveResult) {
	defer r.G().CVTrace(ctx, VLog1, fmt.Sprintf("Resolver#resolveURLViaServerLookup(input = %q)", input), func() error { return res.err })()

	var key, val string
	var ares *APIRes
	var l int

	if au.IsKeybase() {
		res.queriedKbUsername = au.GetValue()
	} else if au.IsUID() {
		res.queriedByUID = true
	}

	if key, val, res.err = au.ToLookup(); res.err != nil {
		return
	}

	ha := HTTPArgsFromKeyValuePair(key, S{val})
	ha.Add("multi", I{1})
	fields := "basics"
	if withBody {
		fields += ",public_keys,pictures"
	}
	ha.Add("fields", S{fields})
	ares, res.err = r.G().API.Get(APIArg{
		Endpoint:       "user/lookup",
		NeedSession:    false,
		Args:           ha,
		AppStatusCodes: []int{SCOk, SCNotFound, SCDeleted},
		NetContext:     ctx,
	})

	if res.err != nil {
		r.G().Log.CDebugf(ctx, "API user/lookup %q error: %s", input, res.err)
		return
	}
	switch ares.AppStatus.Code {
	case SCNotFound:
		r.G().Log.CDebugf(ctx, "API user/lookup %q not found", input)
		res.err = NotFoundError{}
		return
	case SCDeleted:
		r.G().Log.CDebugf(ctx, "API user/lookup %q deleted", input)
		res.err = DeletedError{Msg: fmt.Sprintf("user %q deleted", input)}
		return
	}

	var them *jsonw.Wrapper
	if them, res.err = ares.Body.AtKey("them").ToArray(); res.err != nil {
		return
	}

	if l, res.err = them.Len(); res.err != nil {
		return
	}

	if l == 0 {
		res.err = ResolutionError{Input: input, Msg: "No resolution found"}
	} else if l > 1 {
		res.err = ResolutionError{Input: input, Msg: "Identify is ambiguous"}
	} else {
		res.body = them.AtIndex(0)
		res.uid, res.err = GetUID(res.body.AtKey("id"))
		if res.err == nil {
			res.resolvedKbUsername, res.err = res.body.AtPath("basics.username").GetString()
		}
	}

	return
}

type ResolveCacheStats struct {
	misses          int
	timeouts        int
	mutableTimeouts int
	errorTimeouts   int
	hits            int
	diskGets        int
	diskGetHits     int
	diskGetMisses   int
	diskPuts        int
}

type Resolver struct {
	Contextified
	cache   *ramcache.Ramcache
	Stats   ResolveCacheStats
	NowFunc func() time.Time
}

func (s ResolveCacheStats) Eq(m, t, mt, et, h int) bool {
	return (s.misses == m) && (s.timeouts == t) && (s.mutableTimeouts == mt) && (s.errorTimeouts == et) && (s.hits == h)
}

func (s ResolveCacheStats) EqWithDiskHits(m, t, mt, et, h, dh int) bool {
	return (s.misses == m) && (s.timeouts == t) && (s.mutableTimeouts == mt) && (s.errorTimeouts == et) && (s.hits == h) && (s.diskGetHits == dh)
}

func NewResolver(g *GlobalContext) *Resolver {
	return &Resolver{
		Contextified: NewContextified(g),
		cache:        nil,
		NowFunc:      func() time.Time { return time.Now() },
	}
}

func (r *Resolver) EnableCaching() {
	cache := ramcache.New()
	cache.MaxAge = ResolveCacheMaxAge
	cache.TTL = resolveCacheTTL
	r.cache = cache
}

func (r *Resolver) Shutdown() {
	if r.cache == nil {
		return
	}
	r.cache.Shutdown()
}

func (r *Resolver) getFromMemCache(ctx context.Context, key string, au AssertionURL) (ret *ResolveResult) {
	defer r.G().CVTraceOK(ctx, VLog1, fmt.Sprintf("Resolver#getFromMemCache(%q)", key), func() bool { return ret != nil })()
	if r.cache == nil {
		return nil
	}
	res, _ := r.cache.Get(key)
	if res == nil {
		r.Stats.misses++
		return nil
	}
	rres, ok := res.(*ResolveResult)
	if !ok {
		r.Stats.misses++
		return nil
	}
	// Should never happen, but don't corrupt application state if it does
	if rres.uid.IsNil() {
		r.G().Log.CInfof(ctx, "Resolver#getFromMemCache: nil UID in cache")
		return nil
	}
	now := r.NowFunc()
	if now.Sub(rres.cachedAt) > ResolveCacheMaxAge {
		r.Stats.timeouts++
		return nil
	}
	if rres.mutable && now.Sub(rres.cachedAt) > ResolveCacheMaxAgeMutable {
		r.Stats.mutableTimeouts++
		return nil
	}
	if rres.err != nil && now.Sub(rres.cachedAt) > resolveCacheMaxAgeErrored {
		r.Stats.errorTimeouts++
		return nil
	}
	r.Stats.hits++
	rres.addKeybaseNameIfKnown(au)
	return rres
}

func resolveDbKey(key string) DbKey {
	return DbKey{
		Typ: DBResolveUsernameToUID,
		Key: NewNormalizedUsername(key).String(),
	}
}

func (r *Resolver) putToDiskCache(ctx context.Context, key string, res ResolveResult) {
	r.G().VDL.CLogf(ctx, VLog1, "| Resolver#putToDiskCache (attempt) %+v", res)
	// Only cache immutable resolutions to disk
	if res.mutable {
		return
	}
	// Don't cache errors
	if res.err != nil {
		return
	}
	if res.uid.IsNil() {
		r.G().Log.CWarningf(ctx, "Mistaken UID put to disk cache")
		if r.G().Env.GetDebug() {
			debug.PrintStack()
		}
		return
	}
	r.Stats.diskPuts++
	err := r.G().LocalDb.PutObj(resolveDbKey(key), nil, res.uid)
	if err != nil {
		r.G().Log.CWarningf(ctx, "Cannot put resolve result to disk: %s", err)
		return
	}
	r.G().Log.CDebugf(ctx, "| Resolver#putToDiskCache(%s) -> %v", key, res)
}

// Put receives a copy of a ResolveResult, clears out the body
// to avoid caching data that can go stale, and stores the result.
func (r *Resolver) putToMemCache(key string, res ResolveResult) {
	if r.cache == nil {
		return
	}
	// Don't cache errors
	if res.err != nil {
		return
	}
	if res.uid.IsNil() {
		r.G().Log.Warning("Mistaken UID put to mem cache")
		if r.G().Env.GetDebug() {
			debug.PrintStack()
		}
		return
	}
	res.cachedAt = r.NowFunc()
	res.body = nil // Don't cache body
	r.cache.Set(key, &res)
}
<commit_msg>speed up resolve2(uid) on the critical path of KBFS rekeys (#6161)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"fmt"
	"time"

	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	jsonw "github.com/keybase/go-jsonw"
	"golang.org/x/net/context"
	"runtime/debug"
	"stathat.com/c/ramcache"
)

type ResolveResult struct {
	uid                keybase1.UID
	body               *jsonw.Wrapper
	err                error
	queriedKbUsername  string
	queriedByUID       bool
	resolvedKbUsername string
	cachedAt           time.Time
	mutable            bool
}

func (res ResolveResult) String() string {
	return fmt.Sprintf("{uid:%s err:%s mutable:%v}", res.uid, ErrToOk(res.err), res.mutable)
}

const (
	resolveCacheTTL           = 12 * time.Hour
	ResolveCacheMaxAge        = 12 * time.Hour
	ResolveCacheMaxAgeMutable = 20 * time.Minute
	resolveCacheMaxAgeErrored = 5 * time.Second
)

func (res *ResolveResult) GetUID() keybase1.UID {
	return res.uid
}

func (res *ResolveResult) GetUsername() string {
	return res.resolvedKbUsername
}
func (res *ResolveResult) GetNormalizedUsername() NormalizedUsername {
	return NewNormalizedUsername(res.GetUsername())
}
func (res *ResolveResult) GetNormalizedQueriedUsername() NormalizedUsername {
	return NewNormalizedUsername(res.queriedKbUsername)
}

func (res *ResolveResult) WasKBAssertion() bool {
	return res.queriedKbUsername != "" || res.queriedByUID
}

func (res *ResolveResult) GetError() error {
	return res.err
}

func (res *ResolveResult) GetBody() *jsonw.Wrapper {
	return res.body
}

func (r *Resolver) ResolveWithBody(input string) ResolveResult {
	return r.resolve(input, true)
}

func (r *Resolver) Resolve(input string) ResolveResult {
	return r.resolve(input, false)
}

func (r *Resolver) resolve(input string, withBody bool) (res ResolveResult) {
	defer r.G().Trace(fmt.Sprintf("Resolving username %q", input), func() error { return res.err })()

	var au AssertionURL
	if au, res.err = ParseAssertionURL(r.G().MakeAssertionContext(), input, false); res.err != nil {
		return res
	}
	res = r.resolveURL(context.TODO(), au, input, withBody, false)
	return res
}

func (r *Resolver) ResolveFullExpression(ctx context.Context, input string) (res ResolveResult) {
	return r.resolveFullExpression(ctx, input, false, false)
}

func (r *Resolver) ResolveFullExpressionNeedUsername(ctx context.Context, input string) (res ResolveResult) {
	return r.resolveFullExpression(ctx, input, false, true)
}

func (r *Resolver) ResolveFullExpressionWithBody(ctx context.Context, input string) (res ResolveResult) {
	return r.resolveFullExpression(ctx, input, true, false)
}

func (r *Resolver) resolveFullExpression(ctx context.Context, input string, withBody bool, needUsername bool) (res ResolveResult) {
	defer r.G().CVTrace(ctx, VLog1, fmt.Sprintf("Resolver#resolveFullExpression(%q)", input), func() error { return res.err })()

	var expr AssertionExpression
	expr, res.err = AssertionParseAndOnly(r.G().MakeAssertionContext(), input)
	if res.err != nil {
		return res
	}
	u := FindBestIdentifyComponentURL(expr)
	if u == nil {
		res.err = ResolutionError{Input: input, Msg: "Cannot find a resolvable factor"}
		return res
	}
	return r.resolveURL(ctx, u, input, withBody, needUsername)
}

func (res *ResolveResult) addKeybaseNameIfKnown(au AssertionURL) {
	if au.IsKeybase() && len(res.resolvedKbUsername) == 0 {
		res.resolvedKbUsername = au.GetValue()
	}
}

func (r *Resolver) getFromDiskCache(ctx context.Context, key string, au AssertionURL) (ret *ResolveResult) {
	defer r.G().CVTraceOK(ctx, VLog1, fmt.Sprintf("Resolver#getFromDiskCache(%q)", key), func() bool { return ret != nil })()
	var uid keybase1.UID
	found, err := r.G().LocalDb.GetInto(&uid, resolveDbKey(key))
	r.Stats.diskGets++
	if err != nil {
		r.G().Log.CWarningf(ctx, "Problem fetching resolve result from local DB: %s", err)
		return nil
	}
	if !found {
		r.Stats.diskGetMisses++
		return nil
	}
	if uid.IsNil() {
		r.G().Log.CWarningf(ctx, "nil UID found in disk cache")
		return nil
	}
	r.Stats.diskGetHits++
	return &ResolveResult{uid: uid}
}

func isMutable(au AssertionURL) bool {
	return !(au.IsUID() || au.IsKeybase())
}

func (r *Resolver) getFromUPAKLoader(ctx context.Context, uid keybase1.UID) (ret *ResolveResult) {
	nun, err := r.G().GetUPAKLoader().LookupUsername(ctx, uid)
	if err != nil {
		return nil
	}
	return &ResolveResult{uid: uid, queriedByUID: true, resolvedKbUsername: nun.String(), mutable: false}
}

func (r *Resolver) resolveURL(ctx context.Context, au AssertionURL, input string, withBody bool, needUsername bool) (res ResolveResult) {
	ck := au.CacheKey()

	lock := r.locktab.AcquireOnName(ctx, r.G(), ck)
	defer lock.Release(ctx)

	// Debug succintly what happened in the resolution
	var trace string
	defer func() {
		r.G().Log.CDebugf(ctx, "| Resolver#resolveURL(%s) -> %s [trace:%s]", ck, res, trace)
	}()

	// A standard keybase UID, so it's already resolved... unless we explicitly
	// need it!
	if !needUsername {
		if tmp := au.ToUID(); tmp.Exists() {
			trace += "u"
			return ResolveResult{uid: tmp}
		}
	}

	if p := r.getFromMemCache(ctx, ck, au); p != nil && (!needUsername || len(p.resolvedKbUsername) > 0) {
		trace += "m"
		return *p
	}

	if p := r.getFromDiskCache(ctx, ck, au); p != nil && (!needUsername || len(p.resolvedKbUsername) > 0) {
		p.mutable = isMutable(au)
		r.putToMemCache(ck, *p)
		trace += "d"
		return *p
	}

	// We can check the UPAK loader for the username if we're just mapping a UID to a username.
	if tmp := au.ToUID(); !withBody && tmp.Exists() {
		if p := r.getFromUPAKLoader(ctx, tmp); p != nil {
			trace += "l"
			r.putToMemCache(ck, *p)
			return *p
		}
	}

	trace += "s"
	res = r.resolveURLViaServerLookup(ctx, au, input, withBody)

	// Cache for a shorter period of time if it's not a Keybase identity
	res.mutable = isMutable(au)
	r.putToMemCache(ck, res)

	// We only put to disk cache if it's a Keybase-type assertion. In particular, UIDs
	// are **not** stored to disk.
	if au.IsKeybase() {
		trace += "p"
		r.putToDiskCache(ctx, ck, res)
	}

	return res
}

func (r *Resolver) resolveURLViaServerLookup(ctx context.Context, au AssertionURL, input string, withBody bool) (res ResolveResult) {
	defer r.G().CVTrace(ctx, VLog1, fmt.Sprintf("Resolver#resolveURLViaServerLookup(input = %q)", input), func() error { return res.err })()

	var key, val string
	var ares *APIRes
	var l int

	if au.IsKeybase() {
		res.queriedKbUsername = au.GetValue()
	} else if au.IsUID() {
		res.queriedByUID = true
	}

	if key, val, res.err = au.ToLookup(); res.err != nil {
		return
	}

	ha := HTTPArgsFromKeyValuePair(key, S{val})
	ha.Add("multi", I{1})
	fields := "basics"
	if withBody {
		fields += ",public_keys,pictures"
	}
	ha.Add("fields", S{fields})
	ares, res.err = r.G().API.Get(APIArg{
		Endpoint:       "user/lookup",
		NeedSession:    false,
		Args:           ha,
		AppStatusCodes: []int{SCOk, SCNotFound, SCDeleted},
		NetContext:     ctx,
	})

	if res.err != nil {
		r.G().Log.CDebugf(ctx, "API user/lookup %q error: %s", input, res.err)
		return
	}
	switch ares.AppStatus.Code {
	case SCNotFound:
		r.G().Log.CDebugf(ctx, "API user/lookup %q not found", input)
		res.err = NotFoundError{}
		return
	case SCDeleted:
		r.G().Log.CDebugf(ctx, "API user/lookup %q deleted", input)
		res.err = DeletedError{Msg: fmt.Sprintf("user %q deleted", input)}
		return
	}

	var them *jsonw.Wrapper
	if them, res.err = ares.Body.AtKey("them").ToArray(); res.err != nil {
		return
	}

	if l, res.err = them.Len(); res.err != nil {
		return
	}

	if l == 0 {
		res.err = ResolutionError{Input: input, Msg: "No resolution found"}
	} else if l > 1 {
		res.err = ResolutionError{Input: input, Msg: "Identify is ambiguous"}
	} else {
		res.body = them.AtIndex(0)
		res.uid, res.err = GetUID(res.body.AtKey("id"))
		if res.err == nil {
			res.resolvedKbUsername, res.err = res.body.AtPath("basics.username").GetString()
		}
	}

	return
}

type ResolveCacheStats struct {
	misses          int
	timeouts        int
	mutableTimeouts int
	errorTimeouts   int
	hits            int
	diskGets        int
	diskGetHits     int
	diskGetMisses   int
	diskPuts        int
}

type Resolver struct {
	Contextified
	cache   *ramcache.Ramcache
	Stats   ResolveCacheStats
	NowFunc func() time.Time
	locktab LockTable
}

func (s ResolveCacheStats) Eq(m, t, mt, et, h int) bool {
	return (s.misses == m) && (s.timeouts == t) && (s.mutableTimeouts == mt) && (s.errorTimeouts == et) && (s.hits == h)
}

func (s ResolveCacheStats) EqWithDiskHits(m, t, mt, et, h, dh int) bool {
	return (s.misses == m) && (s.timeouts == t) && (s.mutableTimeouts == mt) && (s.errorTimeouts == et) && (s.hits == h) && (s.diskGetHits == dh)
}

func NewResolver(g *GlobalContext) *Resolver {
	return &Resolver{
		Contextified: NewContextified(g),
		cache:        nil,
		NowFunc:      func() time.Time { return time.Now() },
	}
}

func (r *Resolver) EnableCaching() {
	cache := ramcache.New()
	cache.MaxAge = ResolveCacheMaxAge
	cache.TTL = resolveCacheTTL
	r.cache = cache
}

func (r *Resolver) Shutdown() {
	if r.cache == nil {
		return
	}
	r.cache.Shutdown()
}

func (r *Resolver) getFromMemCache(ctx context.Context, key string, au AssertionURL) (ret *ResolveResult) {
	defer r.G().CVTraceOK(ctx, VLog1, fmt.Sprintf("Resolver#getFromMemCache(%q)", key), func() bool { return ret != nil })()
	if r.cache == nil {
		return nil
	}
	res, _ := r.cache.Get(key)
	if res == nil {
		r.Stats.misses++
		return nil
	}
	rres, ok := res.(*ResolveResult)
	if !ok {
		r.Stats.misses++
		return nil
	}
	// Should never happen, but don't corrupt application state if it does
	if rres.uid.IsNil() {
		r.G().Log.CInfof(ctx, "Resolver#getFromMemCache: nil UID in cache")
		return nil
	}
	now := r.NowFunc()
	if now.Sub(rres.cachedAt) > ResolveCacheMaxAge {
		r.Stats.timeouts++
		return nil
	}
	if rres.mutable && now.Sub(rres.cachedAt) > ResolveCacheMaxAgeMutable {
		r.Stats.mutableTimeouts++
		return nil
	}
	if rres.err != nil && now.Sub(rres.cachedAt) > resolveCacheMaxAgeErrored {
		r.Stats.errorTimeouts++
		return nil
	}
	r.Stats.hits++
	rres.addKeybaseNameIfKnown(au)
	return rres
}

func resolveDbKey(key string) DbKey {
	return DbKey{
		Typ: DBResolveUsernameToUID,
		Key: NewNormalizedUsername(key).String(),
	}
}

func (r *Resolver) putToDiskCache(ctx context.Context, key string, res ResolveResult) {
	r.G().VDL.CLogf(ctx, VLog1, "| Resolver#putToDiskCache (attempt) %+v", res)
	// Only cache immutable resolutions to disk
	if res.mutable {
		return
	}
	// Don't cache errors
	if res.err != nil {
		return
	}
	if res.uid.IsNil() {
		r.G().Log.CWarningf(ctx, "Mistaken UID put to disk cache")
		if r.G().Env.GetDebug() {
			debug.PrintStack()
		}
		return
	}
	r.Stats.diskPuts++
	err := r.G().LocalDb.PutObj(resolveDbKey(key), nil, res.uid)
	if err != nil {
		r.G().Log.CWarningf(ctx, "Cannot put resolve result to disk: %s", err)
		return
	}
	r.G().Log.CDebugf(ctx, "| Resolver#putToDiskCache(%s) -> %v", key, res)
}

// Put receives a copy of a ResolveResult, clears out the body
// to avoid caching data that can go stale, and stores the result.
func (r *Resolver) putToMemCache(key string, res ResolveResult) {
	if r.cache == nil {
		return
	}
	// Don't cache errors
	if res.err != nil {
		return
	}
	if res.uid.IsNil() {
		r.G().Log.Warning("Mistaken UID put to mem cache")
		if r.G().Env.GetDebug() {
			debug.PrintStack()
		}
		return
	}
	res.cachedAt = r.NowFunc()
	res.body = nil // Don't cache body
	r.cache.Set(key, &res)
}
<|endoftext|>


<commit_before>package common

const (
	THRUST_VERSION    = "0.7.5"
	THRUST_GO_VERSION = "0.2.7"
)
<commit_msg>Update Version to 0.3.0 - Stable Windows<commit_after>package common

const (
	THRUST_VERSION    = "0.7.5"
	THRUST_GO_VERSION = "0.3.0"
)
<|endoftext|>


<commit_before>package util

import (
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/mitchellh/go-homedir"
	"gopkg.in/ini.v1"
)

// CLConfig holds our command-line configuration.
var CLConfig struct {
	Output string // (optional) output style ("line" or "CLI")
	Wrap   bool   // (optional) wrap output when CLI style
	Debug  bool   // (optional) prints dubugging info to stdout
	NoExit bool   // (optional) don't return a nonzero exit code on lint errors
}

// Config holds our .vale configuration.
var Config = loadOptions()

// AlertLevels holds the possible values for "level" in an external rule.
var AlertLevels = []string{"suggestion", "warning", "error"}

// LevelToInt allows us to easily compare levels in lint.go.
var LevelToInt = map[string]int{
	"suggestion": 0,
	"warning":    1,
	"error":      2,
}

type config struct {
	Checks        []string                   // All checks to load
	GBaseStyles   []string                   // Global base style
	GChecks       map[string]bool            // Global checks
	MinAlertLevel int                        // Lowest alert level to display
	SBaseStyles   map[string][]string        // Syntax-specific base styles
	SChecks       map[string]map[string]bool // Syntax-specific checks
	StylesPath    string                     // Directory with Rule.yml files
}

func newConfig() *config {
	var cfg config
	cfg.GChecks = make(map[string]bool)
	cfg.SBaseStyles = make(map[string][]string)
	cfg.SChecks = make(map[string]map[string]bool)
	cfg.MinAlertLevel = 0
	cfg.GBaseStyles = []string{"vale"}
	return &cfg
}

func determinePath(configPath string, keyPath string) string {
	sep := string(filepath.Separator)
	abs, _ := filepath.Abs(keyPath)
	rel := strings.TrimRight(keyPath, sep)
	if abs != rel || !strings.Contains(keyPath, sep) {
		// The path was relative
		return filepath.Join(configPath, keyPath)
	}
	return abs
}

// loadConfig loads the .vale file. It checks the current directory up to the
// user's home directory, stopping on the first occurrence of a .vale or _vale
// file.
func loadConfig(names []string) (*ini.File, string, error) {
	var configPath, dir string
	var iniFile *ini.File
	var err error

	count := 0
	for configPath == "" && count < 6 {
		if count == 0 {
			dir, _ = os.Getwd()
		} else {
			dir = filepath.Dir(dir)
		}
		for _, name := range names {
			loc := path.Join(dir, name)
			if FileExists(loc) {
				configPath = loc
				break
			}
		}
		count++
	}

	if configPath == "" {
		configPath, _ = homedir.Dir()
	}
	iniFile, err = ini.Load(configPath)
	return iniFile, dir, err
}

// loadOptions reads the .vale file.
func loadOptions() config {
	cfg := newConfig()
	uCfg, path, err := loadConfig([]string{".vale", "_vale"})
	if err != nil {
		return *cfg
	}

	core := uCfg.Section("")
	global := uCfg.Section("*")

	// Default settings
	for _, k := range core.KeyStrings() {
		if k == "StylesPath" {
			cfg.StylesPath = determinePath(path, core.Key(k).MustString(""))
		} else if k == "MinAlertLevel" {
			level := core.Key(k).In("info", AlertLevels)
			cfg.MinAlertLevel = LevelToInt[level]
		}
	}

	// Global settings
	cfg.GBaseStyles = global.Key("BasedOnStyles").Strings(",")
	for _, k := range global.KeyStrings() {
		if k == "BasedOnStyles" {
			continue
		} else {
			cfg.GChecks[k] = global.Key(k).MustBool(false)
			cfg.Checks = append(cfg.Checks, k)
		}
	}

	// Syntax-specific settings
	for _, sec := range uCfg.SectionStrings() {
		if sec == "*" || sec == "DEFAULT" {
			continue
		}
		syntaxOpts := make(map[string]bool)
		for _, k := range uCfg.Section(sec).KeyStrings() {
			if k == "BasedOnStyles" {
				cfg.SBaseStyles[sec] = uCfg.Section(sec).Key(k).Strings(",")
			} else {
				syntaxOpts[k] = uCfg.Section(sec).Key(k).MustBool(false)
				cfg.Checks = append(cfg.Checks, k)
			}
		}
		cfg.SChecks[sec] = syntaxOpts
	}

	return *cfg
}
<commit_msg>refactor: make suggestion the default level<commit_after>package util

import (
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/mitchellh/go-homedir"
	"gopkg.in/ini.v1"
)

// CLConfig holds our command-line configuration.
var CLConfig struct {
	Output string // (optional) output style ("line" or "CLI")
	Wrap   bool   // (optional) wrap output when CLI style
	Debug  bool   // (optional) prints dubugging info to stdout
	NoExit bool   // (optional) don't return a nonzero exit code on lint errors
}

// Config holds our .vale configuration.
var Config = loadOptions()

// AlertLevels holds the possible values for "level" in an external rule.
var AlertLevels = []string{"suggestion", "warning", "error"}

// LevelToInt allows us to easily compare levels in lint.go.
var LevelToInt = map[string]int{
	"suggestion": 0,
	"warning":    1,
	"error":      2,
}

type config struct {
	Checks        []string                   // All checks to load
	GBaseStyles   []string                   // Global base style
	GChecks       map[string]bool            // Global checks
	MinAlertLevel int                        // Lowest alert level to display
	SBaseStyles   map[string][]string        // Syntax-specific base styles
	SChecks       map[string]map[string]bool // Syntax-specific checks
	StylesPath    string                     // Directory with Rule.yml files
}

func newConfig() *config {
	var cfg config
	cfg.GChecks = make(map[string]bool)
	cfg.SBaseStyles = make(map[string][]string)
	cfg.SChecks = make(map[string]map[string]bool)
	cfg.MinAlertLevel = 0
	cfg.GBaseStyles = []string{"vale"}
	return &cfg
}

func determinePath(configPath string, keyPath string) string {
	sep := string(filepath.Separator)
	abs, _ := filepath.Abs(keyPath)
	rel := strings.TrimRight(keyPath, sep)
	if abs != rel || !strings.Contains(keyPath, sep) {
		// The path was relative
		return filepath.Join(configPath, keyPath)
	}
	return abs
}

// loadConfig loads the .vale file. It checks the current directory up to the
// user's home directory, stopping on the first occurrence of a .vale or _vale
// file.
func loadConfig(names []string) (*ini.File, string, error) {
	var configPath, dir string
	var iniFile *ini.File
	var err error

	count := 0
	for configPath == "" && count < 6 {
		if count == 0 {
			dir, _ = os.Getwd()
		} else {
			dir = filepath.Dir(dir)
		}
		for _, name := range names {
			loc := path.Join(dir, name)
			if FileExists(loc) {
				configPath = loc
				break
			}
		}
		count++
	}

	if configPath == "" {
		configPath, _ = homedir.Dir()
	}
	iniFile, err = ini.Load(configPath)
	return iniFile, dir, err
}

// loadOptions reads the .vale file.
func loadOptions() config {
	cfg := newConfig()
	uCfg, path, err := loadConfig([]string{".vale", "_vale"})
	if err != nil {
		return *cfg
	}

	core := uCfg.Section("")
	global := uCfg.Section("*")

	// Default settings
	for _, k := range core.KeyStrings() {
		if k == "StylesPath" {
			cfg.StylesPath = determinePath(path, core.Key(k).MustString(""))
		} else if k == "MinAlertLevel" {
			level := core.Key(k).In("suggestion", AlertLevels)
			cfg.MinAlertLevel = LevelToInt[level]
		}
	}

	// Global settings
	cfg.GBaseStyles = global.Key("BasedOnStyles").Strings(",")
	for _, k := range global.KeyStrings() {
		if k == "BasedOnStyles" {
			continue
		} else {
			cfg.GChecks[k] = global.Key(k).MustBool(false)
			cfg.Checks = append(cfg.Checks, k)
		}
	}

	// Syntax-specific settings
	for _, sec := range uCfg.SectionStrings() {
		if sec == "*" || sec == "DEFAULT" {
			continue
		}
		syntaxOpts := make(map[string]bool)
		for _, k := range uCfg.Section(sec).KeyStrings() {
			if k == "BasedOnStyles" {
				cfg.SBaseStyles[sec] = uCfg.Section(sec).Key(k).Strings(",")
			} else {
				syntaxOpts[k] = uCfg.Section(sec).Key(k).MustBool(false)
				cfg.Checks = append(cfg.Checks, k)
			}
		}
		cfg.SChecks[sec] = syntaxOpts
	}

	return *cfg
}
<|endoftext|>


<commit_before>// Compare whether the first is greater than, less than, or equal to the second.
//
// The values are either one of the builtin types or their slice, and their
// types MUST be the same, or panic.
//
package compare

import "strings"

// Compare whether v1 is greater than v2.
// Return a positive integer if greater, 0 if equal, a negative if less.
//
// v1 and v2 may be a byte, rune, int, uint, int8, int16, int32, int64,
// uint8, uint16, uint32, uint64, float32, float64, string, or their slice,
// or a struct implementing the interface of Comparer.
func Compare(v1, v2 interface{}) int {
	if _v1, ok := v1.(Comparer); ok {
		return _v1.Compare(v2)
	}

	var first, second float64
	switch _v1 := v1.(type) {
	case int:
		if _v2, ok := v2.(int); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case uint:
		if _v2, ok := v2.(uint); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case int32:
		if _v2, ok := v2.(int32); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case uint32:
		if _v2, ok := v2.(uint32); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case int16:
		if _v2, ok := v2.(int16); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case uint16:
		if _v2, ok := v2.(uint16); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case int64:
		if _v2, ok := v2.(int64); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case uint64:
		if _v2, ok := v2.(uint64); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case int8:
		if _v2, ok := v2.(int8); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case uint8:
		if _v2, ok := v2.(uint8); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case float32:
		if _v2, ok := v2.(float32); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case float64:
		if _v2, ok := v2.(float64); !ok {
			panic("Type is not the same")
		} else {
			first, second = _v1, _v2
		}
	case string:
		if _v2, ok := v2.(string); !ok {
			panic("Type is not the same")
		} else {
			return strings.Compare(_v1, _v2)
		}
	default:
		return CompareSlice(v1, v2)
	}

	if first > second {
		return 1
	} else if first < second {
		return -1
	} else {
		return 0
	}
}

// Same as Compare, but return true if v1 is less than v2, or return false.
func LT(v1, v2 interface{}) bool {
	return Compare(v1, v2) < 0
}

// Same as LT, but greater than.
func GT(v1, v2 interface{}) bool {
	return Compare(v1, v2) > 0
}

// Same as LT, but equal to.
func EQ(v1, v2 interface{}) bool {
	return Compare(v1, v2) == 0
}

// Same as LT, but not equal to.
func NE(v1, v2 interface{}) bool {
	return !EQ(v1, v2)
}

// Same as LT, but greater than or equal to.
func GE(v1, v2 interface{}) bool {
	return GT(v1, v2) || EQ(v1, v2)
}

// Same as LT, but less than or equal to.
func LE(v1, v2 interface{}) bool {
	return LT(v1, v2) || EQ(v1, v2)
}
<commit_msg>Update the comment of the package compare<commit_after>// Package compare compares whether the first is greater than, less than,
// or equal to the second.
//
// The values are either one of the builtin types or their slice, and their
// types MUST be the same, or panic.
//
// You should use these functions: Compare, LT, GT, EQ, NE, LE, GE.
//
package compare

import "strings"

// Compare whether v1 is greater than v2.
// Return a positive integer if greater, 0 if equal, a negative if less.
//
// v1 and v2 may be a byte, rune, int, uint, int8, int16, int32, int64,
// uint8, uint16, uint32, uint64, float32, float64, string, or their slice,
// or a struct implementing the interface of Comparer.
func Compare(v1, v2 interface{}) int {
	if _v1, ok := v1.(Comparer); ok {
		return _v1.Compare(v2)
	}

	var first, second float64
	switch _v1 := v1.(type) {
	case int:
		if _v2, ok := v2.(int); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case uint:
		if _v2, ok := v2.(uint); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case int32:
		if _v2, ok := v2.(int32); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case uint32:
		if _v2, ok := v2.(uint32); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case int16:
		if _v2, ok := v2.(int16); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case uint16:
		if _v2, ok := v2.(uint16); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case int64:
		if _v2, ok := v2.(int64); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case uint64:
		if _v2, ok := v2.(uint64); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case int8:
		if _v2, ok := v2.(int8); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case uint8:
		if _v2, ok := v2.(uint8); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case float32:
		if _v2, ok := v2.(float32); !ok {
			panic("Type is not the same")
		} else {
			first, second = float64(_v1), float64(_v2)
		}
	case float64:
		if _v2, ok := v2.(float64); !ok {
			panic("Type is not the same")
		} else {
			first, second = _v1, _v2
		}
	case string:
		if _v2, ok := v2.(string); !ok {
			panic("Type is not the same")
		} else {
			return strings.Compare(_v1, _v2)
		}
	default:
		return CompareSlice(v1, v2)
	}

	if first > second {
		return 1
	} else if first < second {
		return -1
	} else {
		return 0
	}
}

// LT is the same as Compare, but return true if v1 is less than v2, or return false.
func LT(v1, v2 interface{}) bool {
	return Compare(v1, v2) < 0
}

// GT is the same as LT, but greater than.
func GT(v1, v2 interface{}) bool {
	return Compare(v1, v2) > 0
}

// EQ is the same as LT, but equal to.
func EQ(v1, v2 interface{}) bool {
	return Compare(v1, v2) == 0
}

// NE is the same as LT, but not equal to.
func NE(v1, v2 interface{}) bool {
	return !EQ(v1, v2)
}

// GE is the same as LT, but greater than or equal to.
func GE(v1, v2 interface{}) bool {
	return GT(v1, v2) || EQ(v1, v2)
}

// LE is the same as LT, but less than or equal to.
func LE(v1, v2 interface{}) bool {
	return LT(v1, v2) || EQ(v1, v2)
}
<|endoftext|>


<commit_before>package tcp

import (
	"bytes"
	"fmt"
	"net"
	"sync"
	"time"

	log "github.com/cihub/seelog"
	"github.com/pkg/errors"
	"github.com/schollz/croc/v6/src/comm"
	"github.com/schollz/croc/v6/src/logger"
	"github.com/schollz/croc/v6/src/models"
)

type server struct {
	port       string
	debugLevel string
	banner     string
	rooms      roomMap
}

type roomInfo struct {
	first  *comm.Comm
	second *comm.Comm
	opened time.Time
	full   bool
}

type roomMap struct {
	rooms map[string]roomInfo
	sync.Mutex
}

// Run starts a tcp listener, run async
func Run(debugLevel, port string, banner ...string) (err error) {
	s := new(server)
	s.port = port
	s.debugLevel = debugLevel
	if len(banner) > 0 {
		s.banner = banner[0]
	}
	return s.start()
}

func (s *server) start() (err error) {
	logger.SetLogLevel(s.debugLevel)
	s.rooms.Lock()
	s.rooms.rooms = make(map[string]roomInfo)
	s.rooms.Unlock()

	// delete old rooms
	go func() {
		for {
			time.Sleep(10 * time.Minute)
			s.rooms.Lock()
			for room := range s.rooms.rooms {
				if time.Since(s.rooms.rooms[room].opened) > 3*time.Hour {
					delete(s.rooms.rooms, room)
				}
			}
			s.rooms.Unlock()
		}
	}()

	err = s.run()
	if err != nil {
		log.Error(err)
	}
	return
}

func (s *server) run() (err error) {
	log.Debugf("starting TCP server on " + s.port)
	server, err := net.Listen("tcp", ":"+s.port)
	if err != nil {
		return errors.Wrap(err, "Error listening on :"+s.port)
	}
	defer server.Close()
	// spawn a new goroutine whenever a client connects
	for {
		connection, err := server.Accept()
		if err != nil {
			return errors.Wrap(err, "problem accepting connection")
		}
		log.Debugf("client %s connected", connection.RemoteAddr().String())
		go func(port string, connection net.Conn) {
			errCommunication := s.clientCommuncation(port, comm.New(connection))
			if errCommunication != nil {
				log.Warnf("relay-%s: %s", connection.RemoteAddr().String(), errCommunication.Error())
			}
		}(s.port, connection)
	}
}

func (s *server) clientCommuncation(port string, c *comm.Comm) (err error) {
	// send ok to tell client they are connected
	log.Debug("sending " + s.banner)
	err = c.Send([]byte(s.banner))
	if err != nil {
		return
	}

	// wait for client to tell me which room they want
	log.Debug("waiting for answer")
	roomBytes, err := c.Receive()
	if err != nil {
		return
	}
	room := string(roomBytes)

	s.rooms.Lock()
	// create the room if it is new
	if _, ok := s.rooms.rooms[room]; !ok {
		s.rooms.rooms[room] = roomInfo{
			first:  c,
			opened: time.Now(),
		}
		s.rooms.Unlock()
		// tell the client that they got the room
		err = c.Send([]byte("ok"))
		if err != nil {
			log.Error(err)
			s.deleteRoom(room)
			return
		}
		log.Debugf("room %s has 1", room)
		return nil
	}
	if s.rooms.rooms[room].full {
		s.rooms.Unlock()
		err = c.Send([]byte("room full"))
		if err != nil {
			log.Error(err)
			s.deleteRoom(room)
			return
		}
		return nil
	}
	log.Debugf("room %s has 2", room)
	s.rooms.rooms[room] = roomInfo{
		first:  s.rooms.rooms[room].first,
		second: c,
		opened: s.rooms.rooms[room].opened,
		full:   true,
	}
	otherConnection := s.rooms.rooms[room].first
	s.rooms.Unlock()

	// second connection is the sender, time to staple connections
	var wg sync.WaitGroup
	wg.Add(1)

	// start piping
	go func(com1, com2 *comm.Comm, wg *sync.WaitGroup) {
		log.Debug("starting pipes")
		pipe(com1.Connection(), com2.Connection())
		wg.Done()
		log.Debug("done piping")
	}(otherConnection, c, &wg)

	// tell the sender everything is ready
	err = c.Send([]byte("ok"))
	if err != nil {
		s.deleteRoom(room)
		return
	}
	wg.Wait()

	// delete room
	s.deleteRoom(room)
	return nil
}

func (s *server) deleteRoom(room string) {
	s.rooms.Lock()
	defer s.rooms.Unlock()
	if _, ok := s.rooms.rooms[room]; !ok {
		return
	}
	log.Debugf("deleting room: %s", room)
	s.rooms.rooms[room].first.Close()
	s.rooms.rooms[room].second.Close()
	s.rooms.rooms[room] = roomInfo{first: nil, second: nil}
	delete(s.rooms.rooms, room)

}

// chanFromConn creates a channel from a Conn object, and sends everything it
//  Read()s from the socket to the channel.
func chanFromConn(conn net.Conn) chan []byte {
	c := make(chan []byte, 1)

	go func() {
		b := make([]byte, models.TCP_BUFFER_SIZE)

		for {
			n, err := conn.Read(b)
			if n > 0 {
				res := make([]byte, n)
				// Copy the buffer so it doesn't get changed while read by the recipient.
				copy(res, b[:n])
				c <- res
			}
			if err != nil {
				log.Debug(err)
				c <- nil
				break
			}
		}
		log.Debug("exiting")
	}()

	return c
}

// pipe creates a full-duplex pipe between the two sockets and
// transfers data from one to the other.
func pipe(conn1 net.Conn, conn2 net.Conn) {
	chan1 := chanFromConn(conn1)
	chan2 := chanFromConn(conn2)

	for {
		select {
		case b1 := <-chan1:
			if b1 == nil {
				return
			}
			conn2.Write(b1)

		case b2 := <-chan2:
			if b2 == nil {
				return
			}
			conn1.Write(b2)
		}
	}
}

func ConnectToTCPServer(address, room string) (c *comm.Comm, banner string, err error) {
	c, err = comm.NewConnection(address)
	if err != nil {
		return
	}
	log.Debug("waiting for first ok")
	data, err := c.Receive()
	if err != nil {
		return
	}
	banner = string(data)
	log.Debug("sending room")
	err = c.Send([]byte(room))
	if err != nil {
		return
	}
	log.Debug("waiting for room confirmation")
	data, err = c.Receive()
	if err != nil {
		return
	}
	if !bytes.Equal(data, []byte("ok")) {
		err = fmt.Errorf("got bad response: %s", data)
		return
	}
	log.Debug("all set")
	return
}
<commit_msg>change message<commit_after>package tcp

import (
	"bytes"
	"fmt"
	"net"
	"sync"
	"time"

	log "github.com/cihub/seelog"
	"github.com/pkg/errors"
	"github.com/schollz/croc/v6/src/comm"
	"github.com/schollz/croc/v6/src/logger"
	"github.com/schollz/croc/v6/src/models"
)

type server struct {
	port       string
	debugLevel string
	banner     string
	rooms      roomMap
}

type roomInfo struct {
	first  *comm.Comm
	second *comm.Comm
	opened time.Time
	full   bool
}

type roomMap struct {
	rooms map[string]roomInfo
	sync.Mutex
}

// Run starts a tcp listener, run async
func Run(debugLevel, port string, banner ...string) (err error) {
	s := new(server)
	s.port = port
	s.debugLevel = debugLevel
	if len(banner) > 0 {
		s.banner = banner[0]
	}
	return s.start()
}

func (s *server) start() (err error) {
	logger.SetLogLevel(s.debugLevel)
	s.rooms.Lock()
	s.rooms.rooms = make(map[string]roomInfo)
	s.rooms.Unlock()

	// delete old rooms
	go func() {
		for {
			time.Sleep(10 * time.Minute)
			s.rooms.Lock()
			for room := range s.rooms.rooms {
				if time.Since(s.rooms.rooms[room].opened) > 3*time.Hour {
					delete(s.rooms.rooms, room)
				}
			}
			s.rooms.Unlock()
		}
	}()

	err = s.run()
	if err != nil {
		log.Error(err)
	}
	return
}

func (s *server) run() (err error) {
	log.Debugf("starting TCP server on " + s.port)
	server, err := net.Listen("tcp", ":"+s.port)
	if err != nil {
		return errors.Wrap(err, "Error listening on :"+s.port)
	}
	defer server.Close()
	// spawn a new goroutine whenever a client connects
	for {
		connection, err := server.Accept()
		if err != nil {
			return errors.Wrap(err, "problem accepting connection")
		}
		log.Debugf("client %s connected", connection.RemoteAddr().String())
		go func(port string, connection net.Conn) {
			errCommunication := s.clientCommuncation(port, comm.New(connection))
			if errCommunication != nil {
				log.Warnf("relay-%s: %s", connection.RemoteAddr().String(), errCommunication.Error())
			}
		}(s.port, connection)
	}
}

func (s *server) clientCommuncation(port string, c *comm.Comm) (err error) {
	// send ok to tell client they are connected
	log.Debugf("sending '%s'", s.banner)
	err = c.Send([]byte(s.banner))
	if err != nil {
		return
	}

	// wait for client to tell me which room they want
	log.Debug("waiting for answer")
	roomBytes, err := c.Receive()
	if err != nil {
		return
	}
	room := string(roomBytes)

	s.rooms.Lock()
	// create the room if it is new
	if _, ok := s.rooms.rooms[room]; !ok {
		s.rooms.rooms[room] = roomInfo{
			first:  c,
			opened: time.Now(),
		}
		s.rooms.Unlock()
		// tell the client that they got the room
		err = c.Send([]byte("ok"))
		if err != nil {
			log.Error(err)
			s.deleteRoom(room)
			return
		}
		log.Debugf("room %s has 1", room)
		return nil
	}
	if s.rooms.rooms[room].full {
		s.rooms.Unlock()
		err = c.Send([]byte("room full"))
		if err != nil {
			log.Error(err)
			s.deleteRoom(room)
			return
		}
		return nil
	}
	log.Debugf("room %s has 2", room)
	s.rooms.rooms[room] = roomInfo{
		first:  s.rooms.rooms[room].first,
		second: c,
		opened: s.rooms.rooms[room].opened,
		full:   true,
	}
	otherConnection := s.rooms.rooms[room].first
	s.rooms.Unlock()

	// second connection is the sender, time to staple connections
	var wg sync.WaitGroup
	wg.Add(1)

	// start piping
	go func(com1, com2 *comm.Comm, wg *sync.WaitGroup) {
		log.Debug("starting pipes")
		pipe(com1.Connection(), com2.Connection())
		wg.Done()
		log.Debug("done piping")
	}(otherConnection, c, &wg)

	// tell the sender everything is ready
	err = c.Send([]byte("ok"))
	if err != nil {
		s.deleteRoom(room)
		return
	}
	wg.Wait()

	// delete room
	s.deleteRoom(room)
	return nil
}

func (s *server) deleteRoom(room string) {
	s.rooms.Lock()
	defer s.rooms.Unlock()
	if _, ok := s.rooms.rooms[room]; !ok {
		return
	}
	log.Debugf("deleting room: %s", room)
	s.rooms.rooms[room].first.Close()
	s.rooms.rooms[room].second.Close()
	s.rooms.rooms[room] = roomInfo{first: nil, second: nil}
	delete(s.rooms.rooms, room)

}

// chanFromConn creates a channel from a Conn object, and sends everything it
//  Read()s from the socket to the channel.
func chanFromConn(conn net.Conn) chan []byte {
	c := make(chan []byte, 1)

	go func() {
		b := make([]byte, models.TCP_BUFFER_SIZE)

		for {
			n, err := conn.Read(b)
			if n > 0 {
				res := make([]byte, n)
				// Copy the buffer so it doesn't get changed while read by the recipient.
				copy(res, b[:n])
				c <- res
			}
			if err != nil {
				log.Debug(err)
				c <- nil
				break
			}
		}
		log.Debug("exiting")
	}()

	return c
}

// pipe creates a full-duplex pipe between the two sockets and
// transfers data from one to the other.
func pipe(conn1 net.Conn, conn2 net.Conn) {
	chan1 := chanFromConn(conn1)
	chan2 := chanFromConn(conn2)

	for {
		select {
		case b1 := <-chan1:
			if b1 == nil {
				return
			}
			conn2.Write(b1)

		case b2 := <-chan2:
			if b2 == nil {
				return
			}
			conn1.Write(b2)
		}
	}
}

func ConnectToTCPServer(address, room string) (c *comm.Comm, banner string, err error) {
	c, err = comm.NewConnection(address)
	if err != nil {
		return
	}
	log.Debug("waiting for first ok")
	data, err := c.Receive()
	if err != nil {
		return
	}
	banner = string(data)
	log.Debug("sending room")
	err = c.Send([]byte(room))
	if err != nil {
		return
	}
	log.Debug("waiting for room confirmation")
	data, err = c.Receive()
	if err != nil {
		return
	}
	if !bytes.Equal(data, []byte("ok")) {
		err = fmt.Errorf("got bad response: %s", data)
		return
	}
	log.Debug("all set")
	return
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/centrifugal/centrifugo/logger"

	"github.com/garyburd/redigo/redis"
)

type redisEngine struct {
	app  *application
	pool *redis.Pool
	psc  redis.PubSubConn
}

func newRedisEngine(app *application, host, port, password, db, url string, api bool) *redisEngine {
	server := host + ":" + port
	pool := newPool(server, password, db)
	conn := pool.Get()
	psc := redis.PubSubConn{conn}
	return &redisEngine{
		app:  app,
		pool: pool,
		psc:  psc,
	}
}

func newPool(server, password, db string) *redis.Pool {
	return &redis.Pool{
		MaxIdle:     3,
		IdleTimeout: 240 * time.Second,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial("tcp", server)
			if err != nil {
				logger.CRITICAL.Println(err)
				return nil, err
			}
			if password != "" {
				if _, err := c.Do("AUTH", password); err != nil {
					c.Close()
					logger.CRITICAL.Println(err)
					return nil, err
				}
			}
			if _, err := c.Do("SELECT", db); err != nil {
				c.Close()
				logger.CRITICAL.Println(err)
				return nil, err
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}

func (e *redisEngine) getName() string {
	return "Redis"
}

func (e *redisEngine) initialize() error {
	e.psc.Subscribe(e.app.adminChannel)
	e.psc.Subscribe(e.app.controlChannel)
	go func() {
		for {
			switch n := e.psc.Receive().(type) {
			case redis.Message:
				e.app.handleMessage(n.Channel, string(n.Data))
			case redis.PMessage:
			case redis.Subscription:
			case error:
				logger.ERROR.Printf("error: %v\n", n)
			}
		}
	}()
	return nil
}

func (e *redisEngine) publish(channel, message string) error {
	conn := e.pool.Get()
	defer conn.Close()
	_, err := conn.Do("PUBLISH", channel, message)
	return err
}

func (e *redisEngine) subscribe(channel string) error {
	return e.psc.Subscribe(channel)
}

func (e *redisEngine) unsubscribe(channel string) error {
	return e.psc.Unsubscribe(channel)
}

func (e *redisEngine) addPresence(channel, uid string, info interface{}) error {
	return nil
}

func (e *redisEngine) removePresence(channel, uid string) error {
	return nil
}

func (e *redisEngine) getPresence(channel string) (map[string]interface{}, error) {
	return map[string]interface{}{}, nil
}

func (e *redisEngine) addHistoryMessage(channel string, message interface{}) error {
	return nil
}

func (e *redisEngine) getHistory(channel string) ([]interface{}, error) {
	return []interface{}{}, nil
}
<commit_msg>start implementing redis engine<commit_after>package main

import (
	"time"

	"github.com/centrifugal/centrifugo/logger"

	"github.com/garyburd/redigo/redis"
)

type redisEngine struct {
	app  *application
	pool *redis.Pool
	psc  redis.PubSubConn
}

func newRedisEngine(app *application, host, port, password, db, url string, api bool) *redisEngine {
	server := host + ":" + port
	pool := newPool(server, password, db)
	conn := pool.Get()
	psc := redis.PubSubConn{conn}
	return &redisEngine{
		app:  app,
		pool: pool,
		psc:  psc,
	}
}

func newPool(server, password, db string) *redis.Pool {
	return &redis.Pool{
		MaxIdle:     3,
		IdleTimeout: 240 * time.Second,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial("tcp", server)
			if err != nil {
				logger.CRITICAL.Println(err)
				return nil, err
			}
			if password != "" {
				if _, err := c.Do("AUTH", password); err != nil {
					c.Close()
					logger.CRITICAL.Println(err)
					return nil, err
				}
			}
			if _, err := c.Do("SELECT", db); err != nil {
				c.Close()
				logger.CRITICAL.Println(err)
				return nil, err
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}

func (e *redisEngine) getName() string {
	return "Redis"
}

func (e *redisEngine) initialize() error {
	err := e.psc.Subscribe(e.app.adminChannel)
	if err != nil {
		return err
	}
	err = e.psc.Subscribe(e.app.controlChannel)
	if err != nil {
		return err
	}
	go func() {
		for {
			switch n := e.psc.Receive().(type) {
			case redis.Message:
				e.app.handleMessage(n.Channel, string(n.Data))
			case redis.PMessage:
			case redis.Subscription:
			case error:
				logger.ERROR.Printf("error: %v\n", n)
			}
		}
	}()
	return nil
}

func (e *redisEngine) publish(channel, message string) error {
	conn := e.pool.Get()
	defer conn.Close()
	_, err := conn.Do("PUBLISH", channel, message)
	return err
}

func (e *redisEngine) subscribe(channel string) error {
	return e.psc.Subscribe(channel)
}

func (e *redisEngine) unsubscribe(channel string) error {
	return e.psc.Unsubscribe(channel)
}

func (e *redisEngine) addPresence(channel, uid string, info interface{}) error {
	return nil
}

func (e *redisEngine) removePresence(channel, uid string) error {
	return nil
}

func (e *redisEngine) getPresence(channel string) (map[string]interface{}, error) {
	return map[string]interface{}{}, nil
}

func (e *redisEngine) addHistoryMessage(channel string, message interface{}) error {
	return nil
}

func (e *redisEngine) getHistory(channel string) ([]interface{}, error) {
	return []interface{}{}, nil
}
<|endoftext|>


<commit_before>package boardgame

import (
	"encoding/json"
	"errors"
	"strconv"
	"time"
)

//StateStorageRecord is a record representing a state that can be written to
//storage and later returned. It is an encoded json blob, and can be written
//directly to storage with no modification.
type StateStorageRecord json.RawMessage

//MoveStorageRecord is a record representing the Move that was made to get the
//game to its most recent version.
type MoveStorageRecord struct {
	Name      string
	Version   int
	Initiator int
	//The Phase as returned by Delegate.CurrentPhase() for the state the move
	//was in before it was applied.
	Phase int
	//The player index of the proposer of the move.
	Proposer  PlayerIndex
	Timestamp time.Time
	Blob      json.RawMessage
}

func (m *MoveStorageRecord) String() string {
	return m.Name + ": " + strconv.Itoa(m.Version)
}

//Inflate takes a move storage record and turns it into a move associated with
//that game, if possible. Returns nil if not possible.
func (m *MoveStorageRecord) Inflate(game *Game) (Move, error) {

	if game == nil {
		return nil, errors.New("Game was nil")
	}

	move := game.MoveByName(m.Name)

	if move == nil {
		return nil, errors.New("Couldn't find a move with name: " + m.Name)
	}

	if err := json.Unmarshal(m.Blob, move); err != nil {
		return nil, errors.New("Couldn't unmarshal move: " + err.Error())
	}

	move.Info().version = m.Version
	move.Info().initiator = m.Initiator
	move.Info().timestamp = m.Timestamp

	return move, nil
}

//GameStorageRecord is a simple struct with public fields representing the
//important aspects of a game that should be serialized to storage. The fields
//are broken out specifically so that the storage layer can understand these
//properties in queries.
type GameStorageRecord struct {
	//Name is the type of the game, from its manager. Used for sanity
	//checking.
	Name string
	Id   string
	//SecretSalt for this game for things like component Ids. Should never be
	//transmitted to an insecure or untrusted environment; the only way to
	//access it outside this package is via this field, because it must be
	//able to be persisted to and read from storage.
	SecretSalt string `json:",omitempty"`
	Version    int
	Winners    []PlayerIndex
	Finished   bool
	Created    time.Time
	//Modified is updated every time a new move is applied.
	Modified time.Time
	//NumPlayers is the reported number of players when it was created.
	//Primarily for convenience to storage layer so they know how many players
	//are in the game.
	NumPlayers int
	Agents     []string
	Variant    Variant
}

//StorageManager is an interface that anything can implement to handle the
//persistence of Games and States.
type StorageManager interface {
	//State returns the StateWrapper for the game at the given version, or
	//nil.
	State(gameId string, version int) (StateStorageRecord, error)

	//Move returns the MoveStorageRecord for the game at the given version, or
	//nil.
	Move(gameId string, version int) (*MoveStorageRecord, error)

	//Moves is like Move but returns all moves from fromVersion (exclusive) to
	//toVersion (inclusive). If fromVersion == toVersion, should return
	//toVersion. In many storage subsystems this is cheaper than repeated
	//calls to Move.
	Moves(gameId string, fromVersion, toVersion int) ([]*MoveStorageRecord, error)

	//Game fetches the game with the given ID from the store, if it exists.
	Game(id string) (*GameStorageRecord, error)

	//AgentState retrieves the most recent state for the given agent
	AgentState(gameId string, player PlayerIndex) ([]byte, error)

	//SaveGameAndCurrentState stores the game and the current state (at
	//game.Version()) into the store at the same time in a transaction. If
	//Game.Modifiable() is false, storage should fail. Move can be nil (if game.Version() is 0)
	SaveGameAndCurrentState(game *GameStorageRecord, state StateStorageRecord, move *MoveStorageRecord) error

	//SaveAgentState saves the agent state for the given player
	SaveAgentState(gameId string, player PlayerIndex, state []byte) error

	//PlayerMoveApplied is called after a PlayerMove and all of its resulting
	//FixUp moves have been applied. Most StorageManagers don't need to do
	//anything here; it's primarily useful for signaling that a run of moves
	//has been applied, e.g. in the server.
	PlayerMoveApplied(game *GameStorageRecord) error
}
<commit_msg>Fix up documentation on GameStorageRecord. Part of #562.<commit_after>package boardgame

import (
	"encoding/json"
	"errors"
	"strconv"
	"time"
)

//StateStorageRecord is a record representing a state that can be written to
//storage and later returned. It is an encoded json blob, and can be written
//directly to storage with no modification.
type StateStorageRecord json.RawMessage

//MoveStorageRecord is a record representing the Move that was made to get the
//game to its most recent version.
type MoveStorageRecord struct {
	Name      string
	Version   int
	Initiator int
	//The Phase as returned by Delegate.CurrentPhase() for the state the move
	//was in before it was applied.
	Phase int
	//The player index of the proposer of the move.
	Proposer  PlayerIndex
	Timestamp time.Time
	Blob      json.RawMessage
}

func (m *MoveStorageRecord) String() string {
	return m.Name + ": " + strconv.Itoa(m.Version)
}

//Inflate takes a move storage record and turns it into a move associated with
//that game, if possible. Returns nil if not possible.
func (m *MoveStorageRecord) Inflate(game *Game) (Move, error) {

	if game == nil {
		return nil, errors.New("Game was nil")
	}

	move := game.MoveByName(m.Name)

	if move == nil {
		return nil, errors.New("Couldn't find a move with name: " + m.Name)
	}

	if err := json.Unmarshal(m.Blob, move); err != nil {
		return nil, errors.New("Couldn't unmarshal move: " + err.Error())
	}

	move.Info().version = m.Version
	move.Info().initiator = m.Initiator
	move.Info().timestamp = m.Timestamp

	return move, nil
}

//GameStorageRecord is a simple struct with public fields representing the
//important aspects of a game that should be serialized to storage. The fields
//are broken out specifically so that the storage layer can understand these
//properties in queries. Typically you don't use this struct directly, instead
//getting an inflated version via something like GameManager.ModifiableGame()
//and then using the associated methods on the struct to get at the undelying
//values.
type GameStorageRecord struct {
	//Name is the type of the game, from its manager. Used for sanity
	//checking.
	Name string
	Id   string
	//SecretSalt for this game for things like component Ids. Should never be
	//transmitted to an insecure or untrusted environment; the only way to
	//access it outside this package is via this field, because it must be
	//able to be persisted to and read from storage.
	SecretSalt string `json:",omitempty"`
	Version    int
	Winners    []PlayerIndex
	Finished   bool
	Created    time.Time
	//Modified is updated every time a new move is applied.
	Modified time.Time
	//NumPlayers is the reported number of players when it was created.
	//Primarily for convenience to storage layer so they know how many players
	//are in the game.
	NumPlayers int
	Agents     []string
	Variant    Variant
}

//StorageManager is an interface that anything can implement to handle the
//persistence of Games and States.
type StorageManager interface {
	//State returns the StateWrapper for the game at the given version, or
	//nil.
	State(gameId string, version int) (StateStorageRecord, error)

	//Move returns the MoveStorageRecord for the game at the given version, or
	//nil.
	Move(gameId string, version int) (*MoveStorageRecord, error)

	//Moves is like Move but returns all moves from fromVersion (exclusive) to
	//toVersion (inclusive). If fromVersion == toVersion, should return
	//toVersion. In many storage subsystems this is cheaper than repeated
	//calls to Move.
	Moves(gameId string, fromVersion, toVersion int) ([]*MoveStorageRecord, error)

	//Game fetches the game with the given ID from the store, if it exists.
	Game(id string) (*GameStorageRecord, error)

	//AgentState retrieves the most recent state for the given agent
	AgentState(gameId string, player PlayerIndex) ([]byte, error)

	//SaveGameAndCurrentState stores the game and the current state (at
	//game.Version()) into the store at the same time in a transaction. If
	//Game.Modifiable() is false, storage should fail. Move can be nil (if game.Version() is 0)
	SaveGameAndCurrentState(game *GameStorageRecord, state StateStorageRecord, move *MoveStorageRecord) error

	//SaveAgentState saves the agent state for the given player
	SaveAgentState(gameId string, player PlayerIndex, state []byte) error

	//PlayerMoveApplied is called after a PlayerMove and all of its resulting
	//FixUp moves have been applied. Most StorageManagers don't need to do
	//anything here; it's primarily useful for signaling that a run of moves
	//has been applied, e.g. in the server.
	PlayerMoveApplied(game *GameStorageRecord) error
}
<|endoftext|>


<commit_before>package gom

import (
	"fmt"
	"reflect"
	"strings"
)

type CreateSql func(TableModel, Condition) (string, []interface{})
type TransactionWork func(databaseTx *Db) (int, error)
type SqlFactory interface {
	Insert(TableModel, Condition) (string, []interface{})
	InsertIgnore(TableModel, Condition) (string, []interface{})
	Replace(TableModel, Condition) (string, []interface{})
	Update(TableModel, Condition) (string, []interface{})
	Delete(TableModel, Condition) (string, []interface{})
	Query(TableModel, Condition) (string, []interface{})
}

type TableModel struct {
	Type        reflect.Type
	Value       reflect.Value
	TableName   string
	ColumnNames []string
	Columns     map[string]Column
	Primary     Column
}

func (this TableModel) Clone(value reflect.Value, columnFilters ...string) TableModel {
	var names []string
	if len(columnFilters) > 0 {
		names = columnFilters
	} else {
		names = this.ColumnNames
	}
	return TableModel{this.Type, value, this.TableName, names, this.Columns, this.Primary}
}
func (model TableModel) ColumnsValues() []interface{} {
	var datas []interface{}
	for _, name := range model.ColumnNames {
		column := model.Columns[name]
		var data interface{}
		value := model.Value.FieldByName(column.FieldName)
		if !column.Auto {
			scanner, ok := value.Interface().(IScanner)
			if ok {
				data, _ = scanner.Value()
			} else {
				data = value.Interface()
			}
			datas = append(datas, data)
		}
	}
	return datas
}

type Column struct {
	reflect.Type
	ColumnName string
	FieldName  string
	QueryField string
	IsPrimary  bool
	Auto       bool
}

func (this Column) Clone() Column {
	return Column{this.Type, this.ColumnName, this.FieldName, this.QueryField, this.IsPrimary, this.Auto}
}

type OrderType int

const (
	_ OrderType = iota
	Asc
	Desc
)

type LinkType int

const (
	_ LinkType = iota
	And
	Or
)

type Condition interface {
	Items() []_ConditionItem
	Values() []interface{}
	Pager() Pager
	Order() Order
	NotNull() bool
	Or(sql string, values ...interface{}) Condition
	And(sql string, values ...interface{}) Condition
	AndIn(name string, values ...interface{}) Condition
	OrIn(name string, values ...interface{}) Condition
	Page(index int, size int) Condition
	Limit(index int, size int) Condition
	OrderBy(name string, tp OrderType) Condition
}
type Order interface {
	Name() string
	Type() OrderType
}
type Orders struct {
	MName string
	OrderType
}

func (o Orders) Name() string {
	return o.MName
}
func (o Orders) Type() OrderType {
	return o.OrderType
}

type Pager interface {
	Page() (int, int)
}

type Pagers struct {
	MIndex int
	MSize  int
}

func (p Pagers) Page() (int, int) {
	return p.MIndex, p.MSize
}

type _Condition struct {
	MItems []_ConditionItem
	MOrder Order
	MPager Pager
}
type _ConditionItem struct {
	LinkType LinkType
	States   string
	Values   []interface{}
}

func Cnd(sql string, values ...interface{}) Condition {
	if sql == "" {
		return &_Condition{}
	} else {
		return &_Condition{MItems: []_ConditionItem{{LinkType: And, States: sql, Values: splitArrays(values)}}}
	}
}

func (c *_Condition) Items() []_ConditionItem {
	return c.MItems
}
func (c *_Condition) Values() []interface{} {
	results := []interface{}{}
	length := len(c.MItems)
	if length > 0 {
		for i := 0; i < length; i++ {

			results = append(results, c.MItems[i].Values...)
		}
	}
	return results
}
func (c *_Condition) Pager() Pager {
	return c.MPager
}
func (c *_Condition) Order() Order {
	return c.MOrder
}
func (c *_Condition) NotNull() bool {
	return len(c.MItems) > 0 || c.MOrder != nil || c.MPager != nil
}
func (c *_Condition) And(sql string, values ...interface{}) Condition {
	c.MItems = append(c.MItems, _ConditionItem{LinkType: And, States: sql, Values: splitArrays(values)})
	return c
}
func (c *_Condition) Or(sql string, values ...interface{}) Condition {
	c.MItems = append(c.MItems, _ConditionItem{LinkType: Or, States: sql, Values: splitArrays(values)})
	return c
}
func (c *_Condition) AndIn(name string, values ...interface{}) Condition {
	if len(values) > 0 {
		sql, datas := makeInSql(name, values...)
		c.MItems = append(c.MItems, _ConditionItem{LinkType: And, States: sql, Values: splitArrays(datas)})
	}
	return c
}

func (c *_Condition) OrIn(name string, values ...interface{}) Condition {
	if len(values) > 0 {
		sql, datas := makeInSql(name, values...)
		c.MItems = append(c.MItems, _ConditionItem{LinkType: Or, States: sql, Values: splitArrays(datas)})
	}
	return c
}
func (c *_Condition) Page(index int, size int) Condition {
	c.MPager = Pagers{index * size, size}
	return c
}
func (c *_Condition) Limit(index int, size int) Condition {
	c.MPager = Pagers{index, size}
	return c
}

func (c *_Condition) OrderBy(name string, tp OrderType) Condition {
	c.MOrder = Orders{name, tp}
	return c
}
func splitArrays(values interface{}) []interface{} {
	var results []interface{}
	val := reflect.ValueOf(values)
	_, ptr, sli := getType(values)
	if ptr {
		val = val.Elem()
	}
	if sli {
		lens := val.Len()
		if lens > 0 {
			for i := 0; i < lens; i++ {
				datas := splitArrays(val.Index(i).Interface())
				results = append(results, datas...)
			}
		}
	} else {
		results = append(results, val.Interface())
	}
	return results

}

func makeInSql(name string, values ...interface{}) (string, []interface{}) {
	sql := name + " in ("
	var datas []interface{}
	if len(values) == 0 {
		return "", datas
	}
	datas = splitArrays(values)
	for i := 0; i < len(datas); i++ {
		if i == 0 {
			sql += " ? "
		} else {
			sql += ", ? "
		}
	}
	sql += ")"
	if debug {
		fmt.Println("make in sql was :", sql, datas)
	}
	return sql, datas
}

func (mo TableModel) InsertValues() []interface{} {
	var interfaces []interface{}
	results := reflect.Indirect(reflect.ValueOf(&interfaces))
	for _, name := range mo.ColumnNames {
		if !mo.Primary.Auto || !strings.EqualFold(mo.Primary.ColumnName, name) {
			vars := reflect.ValueOf(mo.Value.FieldByName(mo.Columns[name].FieldName).Interface())
			if results.Kind() == reflect.Ptr {
				results.Set(reflect.Append(results, vars.Addr()))
			} else {
				results.Set(reflect.Append(results, vars))
			}
		}

	}
	return interfaces
}
func (m TableModel) GetPrimary() interface{} {
	return m.Value.FieldByName(m.Primary.FieldName).Interface()
}
func (m TableModel) GetPrimaryCondition() Condition {
	if IsEmpty(m.GetPrimary()) || m.Primary.IsPrimary == false {
		return nil
	} else {
		t := m.Primary.Type
		switch t.Kind() {
		case reflect.Int64, reflect.Int32, reflect.Int, reflect.Int8, reflect.Int16:
			if m.GetPrimary().(int64) == 0 {
				return nil
			}
		}
		switch t.Kind() {
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32:
			if m.GetPrimary().(int64) == 0 {
				return nil
			}
		}
		switch t.Kind() {
		case reflect.Float32, reflect.Float64:
			if m.GetPrimary().(float64) == 0 {
				return nil
			}
		}
		if t.Kind() == reflect.String {
			if len(m.GetPrimary().(string)) == 0 {
				return nil
			}
		}
		return &_Condition{MItems: []_ConditionItem{{And, "`" + m.Primary.ColumnName + "` = ?", []interface{}{m.GetPrimary()}}}}
	}
}
<commit_msg>v1.0.8修复版本重构后Count出错的bug<commit_after>package gom

import (
	"fmt"
	"reflect"
	"strings"
)

type CreateSql func(TableModel, Condition) (string, []interface{})
type TransactionWork func(databaseTx *Db) (int, error)
type SqlFactory interface {
	Insert(TableModel, Condition) (string, []interface{})
	InsertIgnore(TableModel, Condition) (string, []interface{})
	Replace(TableModel, Condition) (string, []interface{})
	Update(TableModel, Condition) (string, []interface{})
	Delete(TableModel, Condition) (string, []interface{})
	Query(TableModel, Condition) (string, []interface{})
}

type TableModel struct {
	Type        reflect.Type
	Value       reflect.Value
	TableName   string
	ColumnNames []string
	Columns     map[string]Column
	Primary     Column
}

func (this TableModel) Clone(value reflect.Value, columnFilters ...string) TableModel {
	var names []string
	if len(columnFilters) > 0 {
		names = columnFilters
	} else {
		names = this.ColumnNames
	}
	return TableModel{this.Type, value, this.TableName, names, this.Columns, this.Primary}
}
func (model TableModel) ColumnsValues() []interface{} {
	var datas []interface{}
	for _, name := range model.ColumnNames {
		column := model.Columns[name]
		var data interface{}
		value := model.Value.FieldByName(column.FieldName)
		if !column.Auto {
			scanner, ok := value.Interface().(IScanner)
			if ok {
				data, _ = scanner.Value()
			} else {
				data = value.Interface()
			}
			datas = append(datas, data)
		}
	}
	return datas
}

type Column struct {
	reflect.Type
	ColumnName string
	FieldName  string
	QueryField string
	IsPrimary  bool
	Auto       bool
}

func (this Column) Clone() Column {
	return Column{this.Type, this.ColumnName, this.FieldName, this.QueryField, this.IsPrimary, this.Auto}
}

type OrderType int

const (
	_ OrderType = iota
	Asc
	Desc
)

type LinkType int

const (
	_ LinkType = iota
	And
	Or
)

type Condition interface {
	Items() []_ConditionItem
	Values() []interface{}
	Pager() Pager
	Order() Order
	NotNull() bool
	Or(sql string, values ...interface{}) Condition
	And(sql string, values ...interface{}) Condition
	AndIn(name string, values ...interface{}) Condition
	OrIn(name string, values ...interface{}) Condition
	Page(index int, size int) Condition
	Limit(index int, size int) Condition
	OrderBy(name string, tp OrderType) Condition
}
type Order interface {
	Name() string
	Type() OrderType
}
type Orders struct {
	MName string
	OrderType
}

func (o Orders) Name() string {
	return o.MName
}
func (o Orders) Type() OrderType {
	return o.OrderType
}

type Pager interface {
	Page() (int, int)
}

type Pagers struct {
	MIndex int
	MSize  int
}

func (p Pagers) Page() (int, int) {
	return p.MIndex, p.MSize
}

type _Condition struct {
	MItems []_ConditionItem
	MOrder Order
	MPager Pager
}
type _ConditionItem struct {
	LinkType LinkType
	States   string
	Values   []interface{}
}

func Cnd(sql string, values ...interface{}) Condition {
	if sql == "" {
		return &_Condition{}
	} else {
		return &_Condition{MItems: []_ConditionItem{{LinkType: And, States: sql, Values: splitArrays(values)}}}
	}
}

func (c *_Condition) Items() []_ConditionItem {
	return c.MItems
}
func (c *_Condition) Values() []interface{} {
	results := []interface{}{}
	length := len(c.MItems)
	if length > 0 {
		for i := 0; i < length; i++ {

			results = append(results, c.MItems[i].Values...)
		}
	}
	return results
}
func (c *_Condition) Pager() Pager {
	return c.MPager
}
func (c *_Condition) Order() Order {
	return c.MOrder
}
func (c *_Condition) NotNull() bool {
	return len(c.MItems) > 0 || c.MOrder != nil || c.MPager != nil
}
func (c *_Condition) And(sql string, values ...interface{}) Condition {
	c.MItems = append(c.MItems, _ConditionItem{LinkType: And, States: sql, Values: splitArrays(values)})
	return c
}
func (c *_Condition) Or(sql string, values ...interface{}) Condition {
	c.MItems = append(c.MItems, _ConditionItem{LinkType: Or, States: sql, Values: splitArrays(values)})
	return c
}
func (c *_Condition) AndIn(name string, values ...interface{}) Condition {
	if len(values) > 0 {
		sql, datas := makeInSql(name, values...)
		c.MItems = append(c.MItems, _ConditionItem{LinkType: And, States: sql, Values: splitArrays(datas)})
	}
	return c
}

func (c *_Condition) OrIn(name string, values ...interface{}) Condition {
	if len(values) > 0 {
		sql, datas := makeInSql(name, values...)
		c.MItems = append(c.MItems, _ConditionItem{LinkType: Or, States: sql, Values: splitArrays(datas)})
	}
	return c
}
func (c *_Condition) Page(index int, size int) Condition {
	c.MPager = Pagers{index * size, size}
	return c
}
func (c *_Condition) Limit(index int, size int) Condition {
	c.MPager = Pagers{index, size}
	return c
}

func (c *_Condition) OrderBy(name string, tp OrderType) Condition {
	c.MOrder = Orders{name, tp}
	return c
}
func splitArrays(values interface{}) []interface{} {
	var results []interface{}
	val := reflect.ValueOf(values)
	_, ptr, sli := getType(values)
	if ptr {
		val = val.Elem()
	}
	if sli {
		lens := val.Len()
		if lens > 0 {
			for i := 0; i < lens; i++ {
				datas := splitArrays(val.Index(i).Interface())
				results = append(results, datas...)
			}
		}
	} else {
		results = append(results, val.Interface())
	}
	return results

}

func makeInSql(name string, values ...interface{}) (string, []interface{}) {
	sql := name + " in ("
	var datas []interface{}
	if len(values) == 0 {
		return "", datas
	}
	datas = splitArrays(values)
	for i := 0; i < len(datas); i++ {
		if i == 0 {
			sql += " ? "
		} else {
			sql += ", ? "
		}
	}
	sql += ")"
	if debug {
		fmt.Println("make in sql was :", sql, datas)
	}
	return sql, datas
}

func (mo TableModel) InsertValues() []interface{} {
	var interfaces []interface{}
	results := reflect.Indirect(reflect.ValueOf(&interfaces))
	for _, name := range mo.ColumnNames {
		if !mo.Primary.Auto || !strings.EqualFold(mo.Primary.ColumnName, name) {
			vars := reflect.ValueOf(mo.Value.FieldByName(mo.Columns[name].FieldName).Interface())
			if results.Kind() == reflect.Ptr {
				results.Set(reflect.Append(results, vars.Addr()))
			} else {
				results.Set(reflect.Append(results, vars))
			}
		}

	}
	return interfaces
}
func (m TableModel) GetPrimary() interface{} {
	return m.Value.FieldByName(m.Primary.FieldName).Interface()
}
func (m TableModel) GetPrimaryCondition() Condition {
	if m.Type.Kind() != reflect.Struct {
		return nil
	}
	if IsEmpty(m.GetPrimary()) || m.Primary.IsPrimary == false {
		return nil
	} else {
		t := m.Primary.Type
		switch t.Kind() {
		case reflect.Int64, reflect.Int32, reflect.Int, reflect.Int8, reflect.Int16:
			if m.GetPrimary().(int64) == 0 {
				return nil
			}
		}
		switch t.Kind() {
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32:
			if m.GetPrimary().(int64) == 0 {
				return nil
			}
		}
		switch t.Kind() {
		case reflect.Float32, reflect.Float64:
			if m.GetPrimary().(float64) == 0 {
				return nil
			}
		}
		if t.Kind() == reflect.String {
			if len(m.GetPrimary().(string)) == 0 {
				return nil
			}
		}
		return &_Condition{MItems: []_ConditionItem{{And, "`" + m.Primary.ColumnName + "` = ?", []interface{}{m.GetPrimary()}}}}
	}
}
<|endoftext|>


<commit_before>package svn

import (
	"encoding/xml"
	"fmt"
	"strconv"

	"github.com/pkg/errors"
)

type Log struct {
	XMLName   xml.Name   `xml:"log"`
	Revisions []Revision `xml:"logentry"`
}

type Revision struct {
	*Commit
	XMLName xml.Name `xml:"logentry"`
	Paths   []Path   `xml:"paths>path"`
	Message string   `xml:"msg"`
}

type Path struct {
	XMLName               xml.Name `xml:"path"`
	TextModifications     bool     `xml:"text-mods,attr"`
	Kind                  string   `xml:"kind,attr"`
	CopyFromPath          *string  `xml:"copyfrom-path,attr,omitempty"`
	CopyFromRevision      *int     `xml:"copyfrom-rev,attr,omitempty"`
	Action                string   `xml:"action,attr"`
	PropertyModifications bool     `xml:"prop-mods,attr"`
	Name                  string   `xml:",chardata"`
}

func GetLog(address string) (*Log, error) {
	log := Log{}

	if err := Execute(&log, "log", "--xml", "--verbose", address); err != nil {
		return nil, errors.Wrapf(err, "failed to get log for %s", address)
	}

	return &log, nil
}

func GetLogRange(address string, start int, end *int) (*Log, error) {
	log := Log{}

	revision := strconv.Itoa(start) + ":"

	if end == nil {
		revision += "HEAD"
	} else {
		revision += strconv.Itoa(*end)
	}

	if err := Execute(&log, "log", "--xml", "--verbose", "--revision", revision, address); err != nil {
		return nil, errors.Wrapf(err, "failed to get log for %s (revision %d)", address, revision)
	}

	return &log, nil
}

func GetRevision(address string, revision int) (*Revision, error) {
	log := Log{}

	if err := Execute(&log, "log", "--xml", "--verbose", "--revision", strconv.Itoa(revision), address); err != nil {
		return nil, errors.Wrapf(err, "failed to get log for %s (revision %d)", address, revision)
	}

	if len(log.Revisions) != 1 {
		return nil, fmt.Errorf("found %d log entries but expected 1 for %s (revison %d)", len(log.Revisions), address, revision)
	}

	return &log.Revisions[0], nil
}
<commit_msg>Add function for retrieving the latest revision<commit_after>package svn

import (
	"encoding/xml"
	"fmt"
	"strconv"

	"github.com/pkg/errors"
)

type Log struct {
	XMLName   xml.Name   `xml:"log"`
	Revisions []Revision `xml:"logentry"`
}

type Revision struct {
	*Commit
	XMLName xml.Name `xml:"logentry"`
	Paths   []Path   `xml:"paths>path"`
	Message string   `xml:"msg"`
}

type Path struct {
	XMLName               xml.Name `xml:"path"`
	TextModifications     bool     `xml:"text-mods,attr"`
	Kind                  string   `xml:"kind,attr"`
	CopyFromPath          *string  `xml:"copyfrom-path,attr,omitempty"`
	CopyFromRevision      *int     `xml:"copyfrom-rev,attr,omitempty"`
	Action                string   `xml:"action,attr"`
	PropertyModifications bool     `xml:"prop-mods,attr"`
	Name                  string   `xml:",chardata"`
}

func GetLog(address string) (*Log, error) {
	log := Log{}

	if err := Execute(&log, "log", "--xml", "--verbose", address); err != nil {
		return nil, errors.Wrapf(err, "failed to get log for %s", address)
	}

	return &log, nil
}

func GetLogRange(address string, start int, end *int) (*Log, error) {
	log := Log{}

	revision := strconv.Itoa(start) + ":"

	if end == nil {
		revision += "HEAD"
	} else {
		revision += strconv.Itoa(*end)
	}

	if err := Execute(&log, "log", "--xml", "--verbose", "--revision", revision, address); err != nil {
		return nil, errors.Wrapf(err, "failed to get log for %s (revision %d)", address, revision)
	}

	return &log, nil
}

func GetRevision(address string, revision int) (*Revision, error) {
	log := Log{}

	if err := Execute(&log, "log", "--xml", "--verbose", "--revision", strconv.Itoa(revision), address); err != nil {
		return nil, errors.Wrapf(err, "failed to get log for %s (revision %d)", address, revision)
	}

	if len(log.Revisions) != 1 {
		return nil, fmt.Errorf("found %d log entries but expected 1 for %s (revison %d)", len(log.Revisions), address, revision)
	}

	return &log.Revisions[0], nil
}

func GetLatestRevision(address string) (*Revision, error) {
	log := Log{}

	if err := Execute(&log, "log", "--xml", "--verbose", "--limit", "1", address); err != nil {
		return nil, errors.Wrapf(err, "failed to get log for %s", address)
	}

	if len(log.Revisions) != 1 {
		return nil, fmt.Errorf("found %d log entries but expected 1 for %s", len(log.Revisions), address)
	}

	return &log.Revisions[0], nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015-2016, RadiantBlue Technologies, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/*
Package utils provides various utilities that are used throughout the project.

Provide functions to return canned responses: StatusOK, StatusBadRequest, and StatusInternalServerError.
*/
package utils

import (
	"fmt"
	"net/http"
	"os"

	"github.com/venicegeo/pzsvc-sdk-go/job"
	"github.com/venicegeo/pzsvc-sdk-go/s3"
)

// FunctionFunc defines the signature of our function creator.
type FunctionFunc func(
	http.ResponseWriter,
	*http.Request,
	*job.OutputMsg,
	job.InputMsg,
)

// MakeFunction wraps the individual PDAL functions.
// Parse the input and output filenames, creating files as needed. Download the
// input data and upload the output data.
func MakeFunction(
	fn func(
		http.ResponseWriter,
		*http.Request,
		*job.OutputMsg,
		job.InputMsg,
		string,
		string,
	),
) FunctionFunc {
	return func(
		w http.ResponseWriter,
		r *http.Request,
		res *job.OutputMsg,
		msg job.InputMsg,
	) {
		var inputName, outputName string
		var fileIn, fileOut *os.File

		// Split the source S3 key string, interpreting the last element as the
		// input filename. Create the input file, throwing 500 on error.
		inputName = s3.ParseFilenameFromKey(msg.Source.Key)
		fileIn, err := os.Create(inputName)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		defer fileIn.Close()

		// If provided, split the destination S3 key string, interpreting the last
		// element as the output filename. Create the output file, throwing 500 on
		// error.
		if len(msg.Destination.Key) > 0 {
			outputName = s3.ParseFilenameFromKey(msg.Destination.Key)
		}

		// Download the source data from S3, throwing 500 on error.
		err = s3.Download(fileIn, msg.Source.Bucket, msg.Source.Key)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		info, _ := os.Stat(outputName)
		fmt.Println(info)
		// if err != nil {
		// 	http.Error(w, err.Error(), http.StatusInternalServerError)
		// 	return
		// }

		// Run the PDAL function.
		fn(w, r, res, msg, inputName, outputName)

		// If an output has been created, upload the destination data to S3,
		// throwing 500 on error.
		if len(msg.Destination.Key) > 0 {
			fileOut, err = os.Open(outputName)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			defer fileOut.Close()
			err = s3.Upload(fileOut, msg.Destination.Bucket, msg.Destination.Key)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		}
	}
}
<commit_msg>Log, not fmt<commit_after>/*
Copyright 2015-2016, RadiantBlue Technologies, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

/*
Package utils provides various utilities that are used throughout the project.

Provide functions to return canned responses: StatusOK, StatusBadRequest, and StatusInternalServerError.
*/
package utils

import (
	"log"
	"net/http"
	"os"

	"github.com/venicegeo/pzsvc-sdk-go/job"
	"github.com/venicegeo/pzsvc-sdk-go/s3"
)

// FunctionFunc defines the signature of our function creator.
type FunctionFunc func(
	http.ResponseWriter,
	*http.Request,
	*job.OutputMsg,
	job.InputMsg,
)

// MakeFunction wraps the individual PDAL functions.
// Parse the input and output filenames, creating files as needed. Download the
// input data and upload the output data.
func MakeFunction(
	fn func(
		http.ResponseWriter,
		*http.Request,
		*job.OutputMsg,
		job.InputMsg,
		string,
		string,
	),
) FunctionFunc {
	return func(
		w http.ResponseWriter,
		r *http.Request,
		res *job.OutputMsg,
		msg job.InputMsg,
	) {
		var inputName, outputName string
		var fileIn, fileOut *os.File

		// Split the source S3 key string, interpreting the last element as the
		// input filename. Create the input file, throwing 500 on error.
		inputName = s3.ParseFilenameFromKey(msg.Source.Key)
		fileIn, err := os.Create(inputName)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		defer fileIn.Close()

		// If provided, split the destination S3 key string, interpreting the last
		// element as the output filename. Create the output file, throwing 500 on
		// error.
		if len(msg.Destination.Key) > 0 {
			outputName = s3.ParseFilenameFromKey(msg.Destination.Key)
		}

		// Download the source data from S3, throwing 500 on error.
		err = s3.Download(fileIn, msg.Source.Bucket, msg.Source.Key)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		info, _ := os.Stat(outputName)
		log.Println(info)
		// if err != nil {
		// 	http.Error(w, err.Error(), http.StatusInternalServerError)
		// 	return
		// }

		// Run the PDAL function.
		fn(w, r, res, msg, inputName, outputName)

		// If an output has been created, upload the destination data to S3,
		// throwing 500 on error.
		if len(msg.Destination.Key) > 0 {
			fileOut, err = os.Open(outputName)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			defer fileOut.Close()
			err = s3.Upload(fileOut, msg.Destination.Bucket, msg.Destination.Key)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2017 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package value

import (
	"bufio"
	"bytes"
	"context"
	"encoding/binary"
	"fmt"
	"io"
	"os"
	"sync"

	"github.com/dgraph-io/badger/y"
	"github.com/dgraph-io/dgraph/x"
	"github.com/pkg/errors"
)

// Values have their first byte being byteData or byteDelete. This helps us distinguish between
// a key that has never been seen and a key that has been explicitly deleted.
const (
	BitDelete       = 1 // Set if the key has been deleted.
	BitValuePointer = 2 // Set if the value is NOT stored directly next to key.
)

type Log struct {
	x.SafeMutex
	fd *os.File
}

type Entry struct {
	Key   []byte
	Meta  byte
	Value []byte
}

type header struct {
	klen uint32
	vlen uint32
}

func (h header) Encode() []byte {
	b := make([]byte, 4+4)
	binary.BigEndian.PutUint32(b[0:4], h.klen)
	binary.BigEndian.PutUint32(b[4:8], h.vlen)
	return b
}

func (h *header) Decode(buf []byte) []byte {
	h.klen = binary.BigEndian.Uint32(buf[0:4])
	h.vlen = binary.BigEndian.Uint32(buf[4:8])
	return buf[8:]
}

type Pointer struct {
	Len    uint32
	Offset uint64
}

// Encode encodes Pointer into byte buffer.
func (p Pointer) Encode(b []byte) []byte {
	y.AssertTrue(len(b) >= 12)
	binary.BigEndian.PutUint32(b[:4], p.Len)
	binary.BigEndian.PutUint64(b[4:12], uint64(p.Offset))
	return b[:12]
}

func (p *Pointer) Decode(b []byte) {
	y.AssertTrue(len(b) >= 12)
	p.Len = binary.BigEndian.Uint32(b[:4])
	p.Offset = binary.BigEndian.Uint64(b[4:12])
}

func (l *Log) Open(fname string) {
	var err error
	l.fd, err = y.OpenSyncedFile(fname)
	y.Check(err)
}

func (l *Log) Replay(offset uint64, fn func(k, v []byte, meta byte)) {
	fmt.Printf("Seeking at offset: %v\n", offset)

	read := func(r *bufio.Reader, buf []byte) error {
		for {
			n, err := r.Read(buf)
			if err != nil {
				return err
			}
			if n == len(buf) {
				return nil
			}
			buf = buf[n:]
		}
	}

	_, err := l.fd.Seek(int64(offset), 0)
	y.Check(err)
	reader := bufio.NewReader(l.fd)

	hbuf := make([]byte, 8)
	var h header
	var count int
	for {
		if err := read(reader, hbuf); err == io.EOF {
			break
		}
		h.Decode(hbuf)
		// fmt.Printf("[%d] Header read: %+v\n", count, h)

		k := make([]byte, h.klen)
		v := make([]byte, h.vlen)

		y.Check(read(reader, k))
		meta, err := reader.ReadByte()
		y.Check(err)
		y.Check(read(reader, v))

		fn(k, v, meta)
		count++
	}
	fmt.Printf("Replayed %d KVs\n", count)
}

var bufPool = sync.Pool{
	New: func() interface{} {
		return &bytes.Buffer{}
	},
}

// Write batches the write of an array of entries to value log.
func (l *Log) Write(entries []Entry) ([]Pointer, error) {
	buf := bufPool.Get().(*bytes.Buffer)
	defer func() {
		buf.Reset()
		bufPool.Put(buf)
	}()

	var h header
	ptrs := make([]Pointer, 0, len(entries))

	for _, e := range entries {
		h.klen = uint32(len(e.Key))
		h.vlen = uint32(len(e.Value))
		header := h.Encode()

		var p Pointer
		p.Len = uint32(len(header)) + h.klen + h.vlen + 1
		p.Offset = uint64(buf.Len())
		ptrs = append(ptrs, p)

		buf.Write(header)
		buf.Write(e.Key)
		buf.WriteByte(e.Meta)
		buf.Write(e.Value)
	}

	l.Lock()
	defer l.Unlock()

	off, err := l.fd.Seek(0, io.SeekEnd)
	y.AssertTrue(off >= 0)
	if err != nil {
		return ptrs, errors.Wrap(err, "Unable to seek")
	}
	for i := range ptrs {
		p := &ptrs[i]
		p.Offset += uint64(off)
	}
	_, err = l.fd.Write(buf.Bytes())
	return ptrs, errors.Wrap(err, "Unable to write to file")
}

// Read reads the value log at a given location.
func (l *Log) Read(ctx context.Context, p Pointer) (e Entry, err error) {
	y.Trace(ctx, "Reading value with pointer: %+v", p)
	defer y.Trace(ctx, "Read done")

	buf := make([]byte, p.Len)
	if _, err := l.fd.ReadAt(buf, int64(p.Offset)); err != nil {
		return e, err
	}
	var h header
	buf = h.Decode(buf)
	e.Key = buf[0:h.klen]
	e.Meta = buf[h.klen]
	buf = buf[h.klen+1:]
	e.Value = buf[0:h.vlen]
	return e, nil
}
<commit_msg>Small fix to remember file offset, instead of retrieving it.<commit_after>/*
 * Copyright 2017 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package value

import (
	"bufio"
	"bytes"
	"context"
	"encoding/binary"
	"fmt"
	"io"
	"os"
	"sync"

	"github.com/dgraph-io/badger/y"
	"github.com/dgraph-io/dgraph/x"
	"github.com/pkg/errors"
)

// Values have their first byte being byteData or byteDelete. This helps us distinguish between
// a key that has never been seen and a key that has been explicitly deleted.
const (
	BitDelete       = 1 // Set if the key has been deleted.
	BitValuePointer = 2 // Set if the value is NOT stored directly next to key.
)

type Log struct {
	x.SafeMutex
	fd     *os.File
	offset int64
}

type Entry struct {
	Key   []byte
	Meta  byte
	Value []byte
}

type header struct {
	klen uint32
	vlen uint32
}

func (h header) Encode() []byte {
	b := make([]byte, 4+4)
	binary.BigEndian.PutUint32(b[0:4], h.klen)
	binary.BigEndian.PutUint32(b[4:8], h.vlen)
	return b
}

func (h *header) Decode(buf []byte) []byte {
	h.klen = binary.BigEndian.Uint32(buf[0:4])
	h.vlen = binary.BigEndian.Uint32(buf[4:8])
	return buf[8:]
}

type Pointer struct {
	Len    uint32
	Offset uint64
}

// Encode encodes Pointer into byte buffer.
func (p Pointer) Encode(b []byte) []byte {
	y.AssertTrue(len(b) >= 12)
	binary.BigEndian.PutUint32(b[:4], p.Len)
	binary.BigEndian.PutUint64(b[4:12], uint64(p.Offset))
	return b[:12]
}

func (p *Pointer) Decode(b []byte) {
	y.AssertTrue(len(b) >= 12)
	p.Len = binary.BigEndian.Uint32(b[:4])
	p.Offset = binary.BigEndian.Uint64(b[4:12])
}

func (l *Log) Open(fname string) {
	var err error
	l.fd, err = y.OpenSyncedFile(fname)
	y.Check(err)
}

func (l *Log) Replay(offset uint64, fn func(k, v []byte, meta byte)) {
	fmt.Printf("Seeking at offset: %v\n", offset)

	read := func(r *bufio.Reader, buf []byte) error {
		for {
			n, err := r.Read(buf)
			if err != nil {
				return err
			}
			if n == len(buf) {
				return nil
			}
			buf = buf[n:]
		}
	}

	_, err := l.fd.Seek(int64(offset), 0)
	y.Check(err)
	reader := bufio.NewReader(l.fd)

	hbuf := make([]byte, 8)
	var h header
	var count int
	for {
		if err := read(reader, hbuf); err == io.EOF {
			break
		}
		h.Decode(hbuf)
		// fmt.Printf("[%d] Header read: %+v\n", count, h)

		k := make([]byte, h.klen)
		v := make([]byte, h.vlen)

		y.Check(read(reader, k))
		meta, err := reader.ReadByte()
		y.Check(err)
		y.Check(read(reader, v))

		fn(k, v, meta)
		count++
	}
	fmt.Printf("Replayed %d KVs\n", count)

	// Seek to the end to start writing.
	l.offset, err := l.fd.Seek(0, io.SeekEnd)
	if err != nil {
		y.Fatalf("Unable to seek to the end. Error: %v", err)
	}
}

var bufPool = sync.Pool{
	New: func() interface{} {
		return &bytes.Buffer{}
	},
}

// Write batches the write of an array of entries to value log.
func (l *Log) Write(entries []Entry) ([]Pointer, error) {
	buf := bufPool.Get().(*bytes.Buffer)
	defer func() {
		buf.Reset()
		bufPool.Put(buf)
	}()

	var h header
	ptrs := make([]Pointer, 0, len(entries))

	for _, e := range entries {
		h.klen = uint32(len(e.Key))
		h.vlen = uint32(len(e.Value))
		header := h.Encode()

		var p Pointer
		p.Len = uint32(len(header)) + h.klen + h.vlen + 1
		p.Offset = uint64(buf.Len())
		ptrs = append(ptrs, p)

		buf.Write(header)
		buf.Write(e.Key)
		buf.WriteByte(e.Meta)
		buf.Write(e.Value)
	}

	l.Lock()
	defer l.Unlock()

	for i := range ptrs {
		p := &ptrs[i]
		p.Offset += uint64(l.offset)
	}
	n, err = l.fd.Write(buf.Bytes())
	l.offset += n
	return ptrs, errors.Wrap(err, "Unable to write to file")
}

// Read reads the value log at a given location.
func (l *Log) Read(ctx context.Context, p Pointer) (e Entry, err error) {
	y.Trace(ctx, "Reading value with pointer: %+v", p)
	defer y.Trace(ctx, "Read done")

	buf := make([]byte, p.Len)
	if _, err := l.fd.ReadAt(buf, int64(p.Offset)); err != nil {
		return e, err
	}
	var h header
	buf = h.Decode(buf)
	e.Key = buf[0:h.klen]
	e.Meta = buf[h.klen]
	buf = buf[h.klen+1:]
	e.Value = buf[0:h.vlen]
	return e, nil
}
<|endoftext|>