<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"fmt"

	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	jsonw "github.com/keybase/go-jsonw"
	"golang.org/x/net/context"
	"runtime/debug"
)

type LoadUserArg struct {
	Contextified
	UID                      keybase1.UID
	Name                     string // Can also be an assertion like foo@twitter
	PublicKeyOptional        bool
	NoCacheResult            bool // currently ignore
	Self                     bool
	ForceReload              bool
	ForcePoll                bool // for cached user load, force a repoll
	StaleOK                  bool // if stale cached versions are OK (for immutable fields)
	AllKeys                  bool
	LoginContext             LoginContext
	AbortIfSigchainUnchanged bool
	ResolveBody              *jsonw.Wrapper // some load paths plumb this through

	// We might have already loaded these if we're falling back from a
	// failed LoadUserPlusKeys load
	MerkleLeaf *MerkleUserLeaf
	SigHints   *SigHints

	// NetContext is the context to build on top of.  We'll make a new
	// Debug Tag for this LoadUser Operation.
	NetContext context.Context
}

func NewLoadUserArg(g *GlobalContext) LoadUserArg {
	return LoadUserArg{Contextified: NewContextified(g)}
}

func NewLoadUserSelfArg(g *GlobalContext) LoadUserArg {
	ret := NewLoadUserArg(g)
	ret.Self = true
	return ret
}

func NewLoadUserForceArg(g *GlobalContext) LoadUserArg {
	arg := NewLoadUserPubOptionalArg(g)
	arg.ForceReload = true
	return arg
}

func NewLoadUserByNameArg(g *GlobalContext, name string) LoadUserArg {
	arg := NewLoadUserArg(g)
	arg.Name = name
	return arg
}

func NewLoadUserByUIDArg(ctx context.Context, g *GlobalContext, uid keybase1.UID) LoadUserArg {
	arg := NewLoadUserArg(g)
	arg.UID = uid
	arg.NetContext = ctx
	return arg
}

func NewLoadUserByUIDForceArg(g *GlobalContext, uid keybase1.UID) LoadUserArg {
	arg := NewLoadUserArg(g)
	arg.UID = uid
	arg.ForceReload = true
	return arg
}

func NewLoadUserPubOptionalArg(g *GlobalContext) LoadUserArg {
	arg := NewLoadUserArg(g)
	arg.PublicKeyOptional = true
	return arg
}

func (arg *LoadUserArg) GetNetContext() context.Context {
	if arg.NetContext != nil {
		return arg.NetContext
	}
	if ctx := arg.G().NetContext; ctx != nil {
		return ctx
	}
	return context.Background()
}

func (arg *LoadUserArg) WithLogTag() context.Context {
	ctx := WithLogTag(arg.GetNetContext(), "LU")
	arg.NetContext = ctx
	arg.SetGlobalContext(arg.G().CloneWithNetContextAndNewLogger(ctx))
	return ctx
}

func (arg *LoadUserArg) checkUIDName() error {
	if arg.UID.Exists() {
		return nil
	}

	if len(arg.Name) == 0 && !arg.Self {
		return fmt.Errorf("no username given to LoadUser")
	}

	if len(arg.Name) > 0 && arg.Self {
		return fmt.Errorf("If loading self, can't provide a username")
	}

	if !arg.Self {
		return nil
	}

	if arg.UID = myUID(arg.G(), arg.LoginContext); arg.UID.IsNil() {
		arg.Name = arg.G().Env.GetUsername().String()
		if len(arg.Name) == 0 {
			return SelfNotFoundError{msg: "could not find UID or username for self"}
		}
	}
	return nil
}

func (arg *LoadUserArg) resolveUID() (ResolveResult, error) {
	var rres ResolveResult
	if arg.UID.Exists() {
		return rres, nil
	}
	if len(arg.Name) == 0 {
		// this won't happen anymore because check moved to
		// checkUIDName() func, but just in case
		return rres, fmt.Errorf("resolveUID: no uid or name")
	}

	if rres = arg.G().Resolver.ResolveWithBody(arg.Name); rres.err != nil {
		return rres, rres.err
	}

	if rres.uid.IsNil() {
		return rres, fmt.Errorf("No resolution for name=%s", arg.Name)
	}

	arg.UID = rres.uid
	return rres, nil
}

// after resolution, check if this is a self load
func (arg *LoadUserArg) checkSelf() {
	if arg.Self {
		return
	}

	myuid := myUID(arg.G(), arg.LoginContext)
	if myuid.Exists() && arg.UID.Exists() && myuid.Equal(arg.UID) {
		arg.Self = true
	}
}

func LoadMe(arg LoadUserArg) (*User, error) {
	arg.Self = true
	return LoadUser(arg)
}

func LoadMeByUID(ctx context.Context, g *GlobalContext, uid keybase1.UID) (*User, error) {
	return LoadMe(NewLoadUserByUIDArg(ctx, g, uid))
}

func LoadUser(arg LoadUserArg) (ret *User, err error) {

	ctx := arg.WithLogTag()
	defer arg.G().CTraceTimed(ctx, fmt.Sprintf("LoadUser(%+v)", arg), func() error { return err })()

	var refresh bool

	if arg.G().VDL.DumpSiteLoadUser() {
		debug.PrintStack()
	}

	// Whatever the reply is, pass along our desired global context
	defer func() {
		if ret != nil {
			ret.SetGlobalContext(arg.G())
			if refresh {
				arg.G().NotifyRouter.HandleUserChanged(ret.GetUID())
			}
		}
	}()

	// make sure we have a uid or a name to load
	if err = arg.checkUIDName(); err != nil {
		return nil, err
	}

	arg.G().Log.CDebugf(ctx, "+ LoadUser(uid=%v, name=%v)", arg.UID, arg.Name)

	// resolve the uid from the name, if necessary
	rres, err := arg.resolveUID()
	if err != nil {
		return nil, err
	}

	// check to see if this is a self load
	arg.checkSelf()

	arg.G().Log.CDebugf(ctx, "| resolved to %s", arg.UID)

	// We can get the user object's body from either the resolution result or
	// if it was plumbed through as a parameter.
	resolveBody := rres.body
	if resolveBody == nil {
		resolveBody = arg.ResolveBody
	}

	// get sig hints from local db in order to populate during merkle leaf lookup
	// They might have already been loaded in.
	var sigHints *SigHints
	if sigHints = arg.SigHints; sigHints == nil {
		sigHints, err = LoadSigHints(ctx, arg.UID, arg.G())
		if err != nil {
			return nil, err
		}
	}

	// load user from local, remote
	ret, refresh, err = loadUser(ctx, arg.G(), arg.UID, resolveBody, sigHints, arg.ForceReload, arg.MerkleLeaf)
	if err != nil {
		return nil, err
	}

	ret.sigHints = sigHints

	// Match the returned User object to the Merkle tree. Also make sure
	// that the username queried for matches the User returned (if it
	// was indeed queried for)
	if err = ret.leaf.MatchUser(ret, arg.UID, rres.GetNormalizedQueriedUsername()); err != nil {
		return ret, err
	}

	if err = ret.LoadSigChains(ctx, arg.AllKeys, &ret.leaf, arg.Self); err != nil {
		return ret, err
	}

	if arg.AbortIfSigchainUnchanged && ret.sigChain().wasFullyCached {
		return nil, nil
	}

	if ret.sigHints != nil && ret.sigHints.dirty {
		refresh = true
	}

	// Proactively cache fetches from remote server to local storage
	if e2 := ret.Store(ctx); e2 != nil {
		arg.G().Log.CWarningf(ctx, "Problem storing user %s: %s", ret.GetName(), e2)
	}

	if ret.HasActiveKey() {
		if err = ret.MakeIDTable(); err != nil {
			return ret, err
		}

		// Check that the user has self-signed only after we
		// consider revocations. See: https://github.com/keybase/go/issues/43
		if err = ret.VerifySelfSig(); err != nil {
			return ret, err
		}

	} else if !arg.PublicKeyOptional {

		var emsg string
		if arg.Self {
			emsg = "You don't have a public key; try `keybase pgp select` or `keybase pgp import` if you have a key; or `keybase pgp gen` if you don't"
		}
		err = NoKeyError{emsg}
	}

	return ret, err
}

func loadUser(ctx context.Context, g *GlobalContext, uid keybase1.UID, resolveBody *jsonw.Wrapper, sigHints *SigHints, force bool, leaf *MerkleUserLeaf) (*User, bool, error) {
	local, err := loadUserFromLocalStorage(ctx, g, uid)
	var refresh bool
	if err != nil {
		g.Log.CWarningf(ctx, "Failed to load %s from storage: %s", uid, err)
	}

	if leaf == nil {
		leaf, err = lookupMerkleLeaf(ctx, g, uid, (local != nil), sigHints)
		if err != nil {
			return nil, refresh, err
		}
	}

	var f1, loadRemote bool

	if local == nil {
		g.Log.CDebugf(ctx, "| No local user stored for %s", uid)
		loadRemote = true
	} else if f1, err = local.CheckBasicsFreshness(leaf.idVersion); err != nil {
		return nil, refresh, err
	} else {
		loadRemote = !f1
		refresh = loadRemote
	}

	g.Log.CDebugf(ctx, "| Freshness: basics=%v; for %s", f1, uid)

	var ret *User
	if !loadRemote && !force {
		ret = local
	} else if ret, err = LoadUserFromServer(ctx, g, uid, resolveBody); err != nil {
		return nil, refresh, err
	}

	if ret == nil {
		return nil, refresh, nil
	}

	if leaf != nil {
		ret.leaf = *leaf
	}
	return ret, refresh, nil
}

func loadUserFromLocalStorage(ctx context.Context, g *GlobalContext, uid keybase1.UID) (u *User, err error) {
	g.Log.CDebugf(ctx, "+ loadUserFromLocalStorage(%s)", uid)
	jw, err := g.LocalDb.Get(DbKeyUID(DBUser, uid))
	if err != nil {
		return nil, err
	}

	if jw == nil {
		g.Log.CDebugf(ctx, "- loadUserFromLocalStorage(%s): Not found", uid)
		return nil, nil
	}

	g.Log.CDebugf(ctx, "| Loaded successfully")

	if u, err = NewUserFromLocalStorage(g, jw); err != nil {
		return nil, err
	}

	if u.id.NotEqual(uid) {
		err = fmt.Errorf("Bad lookup; uid mismatch: %s != %s", uid, u.id)
	}

	g.Log.CDebugf(ctx, "| Loaded username %s (uid=%s)", u.name, uid)
	g.Log.CDebugf(ctx, "- loadUserFromLocalStorage(%s,%s)", u.name, uid)

	return
}

// LoadUserEmails returns emails for logged in user
func LoadUserEmails(g *GlobalContext) (emails []keybase1.Email, err error) {
	uid := g.GetMyUID()
	res, err := g.API.Get(APIArg{
		Endpoint:    "user/lookup",
		NeedSession: true,
		Args: HTTPArgs{
			"uid": UIDArg(uid),
		},
	})
	if err != nil {
		return
	}
	var email string
	var isVerified int
	primary := res.Body.AtKey("them").AtKey("emails").AtKey("primary")
	email, err = primary.AtKey("email").GetString()
	if err != nil {
		return
	}
	isVerified, err = primary.AtKey("is_verified").GetInt()
	if err != nil {
		return
	}
	emails = []keybase1.Email{keybase1.Email{Email: email, IsVerified: isVerified == 1}}
	return
}

func LoadUserFromServer(ctx context.Context, g *GlobalContext, uid keybase1.UID, body *jsonw.Wrapper) (u *User, err error) {
	g.Log.CDebugf(ctx, "+ Load User from server: %s", uid)

	// Res.body might already have been preloaded a a result of a Resolve call earlier.
	if body == nil {
		res, err := g.API.Get(APIArg{
			Endpoint:    "user/lookup",
			NeedSession: false,
			Args: HTTPArgs{
				"uid": UIDArg(uid),
			},
			NetContext: ctx,
		})

		if err != nil {
			return nil, err
		}
		body = res.Body.AtKey("them")
	} else {
		g.Log.CDebugf(ctx, "| Skipped load; got user object previously")
	}

	if u, err = NewUserFromServer(g, body); err != nil {
		return u, err
	}
	g.Log.CDebugf(ctx, "- Load user from server: %s -> %s", uid, ErrToOk(err))

	return u, err
}

func myUID(g *GlobalContext, lctx LoginContext) keybase1.UID {
	if lctx != nil {
		return lctx.LocalSession().GetUID()
	}
	return g.GetMyUID()
}

func lookupMerkleLeaf(ctx context.Context, g *GlobalContext, uid keybase1.UID, localExists bool, sigHints *SigHints) (f *MerkleUserLeaf, err error) {
	if uid.IsNil() {
		err = fmt.Errorf("uid parameter for lookupMerkleLeaf empty")
		return
	}

	q := NewHTTPArgs()
	q.Add("uid", UIDArg(uid))

	f, err = g.MerkleClient.LookupUser(ctx, q, sigHints)
	if err == nil && f == nil && localExists {
		err = fmt.Errorf("User not found in server Merkle tree")
	}

	return
}

func LoadUserPlusKeys(ctx context.Context, g *GlobalContext, uid keybase1.UID) (keybase1.UserPlusKeys, error) {
	return g.GetUPAKLoader().LoadUserPlusKeys(ctx, uid)
}
<commit_msg>fix race condition in debugging (#5642)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"fmt"

	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	jsonw "github.com/keybase/go-jsonw"
	"golang.org/x/net/context"
	"runtime/debug"
)

type LoadUserArg struct {
	Contextified
	UID                      keybase1.UID
	Name                     string // Can also be an assertion like foo@twitter
	PublicKeyOptional        bool
	NoCacheResult            bool // currently ignore
	Self                     bool
	ForceReload              bool
	ForcePoll                bool // for cached user load, force a repoll
	StaleOK                  bool // if stale cached versions are OK (for immutable fields)
	AllKeys                  bool
	LoginContext             LoginContext
	AbortIfSigchainUnchanged bool
	ResolveBody              *jsonw.Wrapper // some load paths plumb this through

	// We might have already loaded these if we're falling back from a
	// failed LoadUserPlusKeys load
	MerkleLeaf *MerkleUserLeaf
	SigHints   *SigHints

	// NetContext is the context to build on top of.  We'll make a new
	// Debug Tag for this LoadUser Operation.
	NetContext context.Context
}

func (arg LoadUserArg) String() string {
	return fmt.Sprintf("{UID:%s Name:%q PublicKeyOptional:%v NoCacheResult:%v Self:%v ForceReload:%v ForcePoll:%v StaleOK:%v AllKeys:%v AbortIfSigchainUnchanged:%v}",
		arg.UID, arg.Name, arg.PublicKeyOptional, arg.NoCacheResult, arg.Self, arg.ForceReload,
		arg.ForcePoll, arg.StaleOK, arg.AllKeys, arg.AbortIfSigchainUnchanged)
}

func NewLoadUserArg(g *GlobalContext) LoadUserArg {
	return LoadUserArg{Contextified: NewContextified(g)}
}

func NewLoadUserSelfArg(g *GlobalContext) LoadUserArg {
	ret := NewLoadUserArg(g)
	ret.Self = true
	return ret
}

func NewLoadUserForceArg(g *GlobalContext) LoadUserArg {
	arg := NewLoadUserPubOptionalArg(g)
	arg.ForceReload = true
	return arg
}

func NewLoadUserByNameArg(g *GlobalContext, name string) LoadUserArg {
	arg := NewLoadUserArg(g)
	arg.Name = name
	return arg
}

func NewLoadUserByUIDArg(ctx context.Context, g *GlobalContext, uid keybase1.UID) LoadUserArg {
	arg := NewLoadUserArg(g)
	arg.UID = uid
	arg.NetContext = ctx
	return arg
}

func NewLoadUserByUIDForceArg(g *GlobalContext, uid keybase1.UID) LoadUserArg {
	arg := NewLoadUserArg(g)
	arg.UID = uid
	arg.ForceReload = true
	return arg
}

func NewLoadUserPubOptionalArg(g *GlobalContext) LoadUserArg {
	arg := NewLoadUserArg(g)
	arg.PublicKeyOptional = true
	return arg
}

func (arg *LoadUserArg) GetNetContext() context.Context {
	if arg.NetContext != nil {
		return arg.NetContext
	}
	if ctx := arg.G().NetContext; ctx != nil {
		return ctx
	}
	return context.Background()
}

func (arg *LoadUserArg) WithLogTag() context.Context {
	ctx := WithLogTag(arg.GetNetContext(), "LU")
	arg.NetContext = ctx
	arg.SetGlobalContext(arg.G().CloneWithNetContextAndNewLogger(ctx))
	return ctx
}

func (arg *LoadUserArg) checkUIDName() error {
	if arg.UID.Exists() {
		return nil
	}

	if len(arg.Name) == 0 && !arg.Self {
		return fmt.Errorf("no username given to LoadUser")
	}

	if len(arg.Name) > 0 && arg.Self {
		return fmt.Errorf("If loading self, can't provide a username")
	}

	if !arg.Self {
		return nil
	}

	if arg.UID = myUID(arg.G(), arg.LoginContext); arg.UID.IsNil() {
		arg.Name = arg.G().Env.GetUsername().String()
		if len(arg.Name) == 0 {
			return SelfNotFoundError{msg: "could not find UID or username for self"}
		}
	}
	return nil
}

func (arg *LoadUserArg) resolveUID() (ResolveResult, error) {
	var rres ResolveResult
	if arg.UID.Exists() {
		return rres, nil
	}
	if len(arg.Name) == 0 {
		// this won't happen anymore because check moved to
		// checkUIDName() func, but just in case
		return rres, fmt.Errorf("resolveUID: no uid or name")
	}

	if rres = arg.G().Resolver.ResolveWithBody(arg.Name); rres.err != nil {
		return rres, rres.err
	}

	if rres.uid.IsNil() {
		return rres, fmt.Errorf("No resolution for name=%s", arg.Name)
	}

	arg.UID = rres.uid
	return rres, nil
}

// after resolution, check if this is a self load
func (arg *LoadUserArg) checkSelf() {
	if arg.Self {
		return
	}

	myuid := myUID(arg.G(), arg.LoginContext)
	if myuid.Exists() && arg.UID.Exists() && myuid.Equal(arg.UID) {
		arg.Self = true
	}
}

func LoadMe(arg LoadUserArg) (*User, error) {
	arg.Self = true
	return LoadUser(arg)
}

func LoadMeByUID(ctx context.Context, g *GlobalContext, uid keybase1.UID) (*User, error) {
	return LoadMe(NewLoadUserByUIDArg(ctx, g, uid))
}

func LoadUser(arg LoadUserArg) (ret *User, err error) {

	ctx := arg.WithLogTag()
	defer arg.G().CTraceTimed(ctx, fmt.Sprintf("LoadUser(%s)", arg), func() error { return err })()

	var refresh bool

	if arg.G().VDL.DumpSiteLoadUser() {
		debug.PrintStack()
	}

	// Whatever the reply is, pass along our desired global context
	defer func() {
		if ret != nil {
			ret.SetGlobalContext(arg.G())
			if refresh {
				arg.G().NotifyRouter.HandleUserChanged(ret.GetUID())
			}
		}
	}()

	// make sure we have a uid or a name to load
	if err = arg.checkUIDName(); err != nil {
		return nil, err
	}

	arg.G().Log.CDebugf(ctx, "+ LoadUser(uid=%v, name=%v)", arg.UID, arg.Name)

	// resolve the uid from the name, if necessary
	rres, err := arg.resolveUID()
	if err != nil {
		return nil, err
	}

	// check to see if this is a self load
	arg.checkSelf()

	arg.G().Log.CDebugf(ctx, "| resolved to %s", arg.UID)

	// We can get the user object's body from either the resolution result or
	// if it was plumbed through as a parameter.
	resolveBody := rres.body
	if resolveBody == nil {
		resolveBody = arg.ResolveBody
	}

	// get sig hints from local db in order to populate during merkle leaf lookup
	// They might have already been loaded in.
	var sigHints *SigHints
	if sigHints = arg.SigHints; sigHints == nil {
		sigHints, err = LoadSigHints(ctx, arg.UID, arg.G())
		if err != nil {
			return nil, err
		}
	}

	// load user from local, remote
	ret, refresh, err = loadUser(ctx, arg.G(), arg.UID, resolveBody, sigHints, arg.ForceReload, arg.MerkleLeaf)
	if err != nil {
		return nil, err
	}

	ret.sigHints = sigHints

	// Match the returned User object to the Merkle tree. Also make sure
	// that the username queried for matches the User returned (if it
	// was indeed queried for)
	if err = ret.leaf.MatchUser(ret, arg.UID, rres.GetNormalizedQueriedUsername()); err != nil {
		return ret, err
	}

	if err = ret.LoadSigChains(ctx, arg.AllKeys, &ret.leaf, arg.Self); err != nil {
		return ret, err
	}

	if arg.AbortIfSigchainUnchanged && ret.sigChain().wasFullyCached {
		return nil, nil
	}

	if ret.sigHints != nil && ret.sigHints.dirty {
		refresh = true
	}

	// Proactively cache fetches from remote server to local storage
	if e2 := ret.Store(ctx); e2 != nil {
		arg.G().Log.CWarningf(ctx, "Problem storing user %s: %s", ret.GetName(), e2)
	}

	if ret.HasActiveKey() {
		if err = ret.MakeIDTable(); err != nil {
			return ret, err
		}

		// Check that the user has self-signed only after we
		// consider revocations. See: https://github.com/keybase/go/issues/43
		if err = ret.VerifySelfSig(); err != nil {
			return ret, err
		}

	} else if !arg.PublicKeyOptional {

		var emsg string
		if arg.Self {
			emsg = "You don't have a public key; try `keybase pgp select` or `keybase pgp import` if you have a key; or `keybase pgp gen` if you don't"
		}
		err = NoKeyError{emsg}
	}

	return ret, err
}

func loadUser(ctx context.Context, g *GlobalContext, uid keybase1.UID, resolveBody *jsonw.Wrapper, sigHints *SigHints, force bool, leaf *MerkleUserLeaf) (*User, bool, error) {
	local, err := loadUserFromLocalStorage(ctx, g, uid)
	var refresh bool
	if err != nil {
		g.Log.CWarningf(ctx, "Failed to load %s from storage: %s", uid, err)
	}

	if leaf == nil {
		leaf, err = lookupMerkleLeaf(ctx, g, uid, (local != nil), sigHints)
		if err != nil {
			return nil, refresh, err
		}
	}

	var f1, loadRemote bool

	if local == nil {
		g.Log.CDebugf(ctx, "| No local user stored for %s", uid)
		loadRemote = true
	} else if f1, err = local.CheckBasicsFreshness(leaf.idVersion); err != nil {
		return nil, refresh, err
	} else {
		loadRemote = !f1
		refresh = loadRemote
	}

	g.Log.CDebugf(ctx, "| Freshness: basics=%v; for %s", f1, uid)

	var ret *User
	if !loadRemote && !force {
		ret = local
	} else if ret, err = LoadUserFromServer(ctx, g, uid, resolveBody); err != nil {
		return nil, refresh, err
	}

	if ret == nil {
		return nil, refresh, nil
	}

	if leaf != nil {
		ret.leaf = *leaf
	}
	return ret, refresh, nil
}

func loadUserFromLocalStorage(ctx context.Context, g *GlobalContext, uid keybase1.UID) (u *User, err error) {
	g.Log.CDebugf(ctx, "+ loadUserFromLocalStorage(%s)", uid)
	jw, err := g.LocalDb.Get(DbKeyUID(DBUser, uid))
	if err != nil {
		return nil, err
	}

	if jw == nil {
		g.Log.CDebugf(ctx, "- loadUserFromLocalStorage(%s): Not found", uid)
		return nil, nil
	}

	g.Log.CDebugf(ctx, "| Loaded successfully")

	if u, err = NewUserFromLocalStorage(g, jw); err != nil {
		return nil, err
	}

	if u.id.NotEqual(uid) {
		err = fmt.Errorf("Bad lookup; uid mismatch: %s != %s", uid, u.id)
	}

	g.Log.CDebugf(ctx, "| Loaded username %s (uid=%s)", u.name, uid)
	g.Log.CDebugf(ctx, "- loadUserFromLocalStorage(%s,%s)", u.name, uid)

	return
}

// LoadUserEmails returns emails for logged in user
func LoadUserEmails(g *GlobalContext) (emails []keybase1.Email, err error) {
	uid := g.GetMyUID()
	res, err := g.API.Get(APIArg{
		Endpoint:    "user/lookup",
		NeedSession: true,
		Args: HTTPArgs{
			"uid": UIDArg(uid),
		},
	})
	if err != nil {
		return
	}
	var email string
	var isVerified int
	primary := res.Body.AtKey("them").AtKey("emails").AtKey("primary")
	email, err = primary.AtKey("email").GetString()
	if err != nil {
		return
	}
	isVerified, err = primary.AtKey("is_verified").GetInt()
	if err != nil {
		return
	}
	emails = []keybase1.Email{keybase1.Email{Email: email, IsVerified: isVerified == 1}}
	return
}

func LoadUserFromServer(ctx context.Context, g *GlobalContext, uid keybase1.UID, body *jsonw.Wrapper) (u *User, err error) {
	g.Log.CDebugf(ctx, "+ Load User from server: %s", uid)

	// Res.body might already have been preloaded a a result of a Resolve call earlier.
	if body == nil {
		res, err := g.API.Get(APIArg{
			Endpoint:    "user/lookup",
			NeedSession: false,
			Args: HTTPArgs{
				"uid": UIDArg(uid),
			},
			NetContext: ctx,
		})

		if err != nil {
			return nil, err
		}
		body = res.Body.AtKey("them")
	} else {
		g.Log.CDebugf(ctx, "| Skipped load; got user object previously")
	}

	if u, err = NewUserFromServer(g, body); err != nil {
		return u, err
	}
	g.Log.CDebugf(ctx, "- Load user from server: %s -> %s", uid, ErrToOk(err))

	return u, err
}

func myUID(g *GlobalContext, lctx LoginContext) keybase1.UID {
	if lctx != nil {
		return lctx.LocalSession().GetUID()
	}
	return g.GetMyUID()
}

func lookupMerkleLeaf(ctx context.Context, g *GlobalContext, uid keybase1.UID, localExists bool, sigHints *SigHints) (f *MerkleUserLeaf, err error) {
	if uid.IsNil() {
		err = fmt.Errorf("uid parameter for lookupMerkleLeaf empty")
		return
	}

	q := NewHTTPArgs()
	q.Add("uid", UIDArg(uid))

	f, err = g.MerkleClient.LookupUser(ctx, q, sigHints)
	if err == nil && f == nil && localExists {
		err = fmt.Errorf("User not found in server Merkle tree")
	}

	return
}

func LoadUserPlusKeys(ctx context.Context, g *GlobalContext, uid keybase1.UID) (keybase1.UserPlusKeys, error) {
	return g.GetUPAKLoader().LoadUserPlusKeys(ctx, uid)
}
<|endoftext|>


<commit_before>package revok

import (
	"context"
	"cred-alert/db"
	"cred-alert/gitclient"
	"cred-alert/scanners"
	"cred-alert/scanners/filescanner"
	"cred-alert/sniff"
	"fmt"
	"io"
	"os"
	"time"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/tedsuo/ifrit"
)

type headCredentialCounter struct {
	logger               lager.Logger
	repositoryRepository db.RepositoryRepository
	clock                clock.Clock
	interval             time.Duration
	gitClient            gitclient.Client
	sniffer              sniff.Sniffer
}

func NewHeadCredentialCounter(
	logger lager.Logger,
	repositoryRepository db.RepositoryRepository,
	clock clock.Clock,
	interval time.Duration,
	gitClient gitclient.Client,
	sniffer sniff.Sniffer,
) ifrit.Runner {
	return &headCredentialCounter{
		logger:               logger,
		repositoryRepository: repositoryRepository,
		clock:                clock,
		interval:             interval,
		gitClient:            gitClient,
		sniffer:              sniffer,
	}
}

func (c *headCredentialCounter) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
	logger := c.logger.Session("head-credential-counter")
	logger.Info("starting")

	close(ready)

	timer := c.clock.NewTicker(c.interval)

	ctx, cancel := context.WithCancel(context.Background())

	c.work(logger, cancel, signals)

	for {
		select {
		case <-timer.C():
			c.work(logger, cancel, signals)
		case <-signals:
			cancel()
			return nil
		case <-ctx.Done():
			return nil
		}
	}
}

func (c *headCredentialCounter) work(
	logger lager.Logger,
	cancel context.CancelFunc,
	signals <-chan os.Signal,
) {
	repositories, err := c.repositoryRepository.All()
	if err != nil {
		logger.Error("failed-getting-all-repositories", err)
	}

	for i := range repositories {
		select {
		case <-signals:
			cancel()
			return
		default:
			repository := repositories[i]
			repoLogger := logger.WithData(lager.Data{
				"ref":  repository.DefaultBranch,
				"path": repository.Path,
			})

			r, w := io.Pipe()

			errCh := make(chan error)
			go func() {
				errCh <- c.gitClient.AllBlobsForRef(repository.Path, fmt.Sprintf("refs/remotes/origin/%s", repository.DefaultBranch), w)
			}()

			var credCount uint
			_ = c.sniffer.Sniff(
				logger,
				filescanner.New(r, ""), // no filename necessary
				func(lager.Logger, scanners.Violation) error {
					credCount++
					return nil
				})

			if err := <-errCh; err != nil {
				repoLogger.Error("failed-to-get-blobs", err)
				continue
			}

			err := c.repositoryRepository.UpdateCredentialCount(&repository, credCount)
			if err != nil {
				repoLogger.Error("failed-to-update-credential-count", err)
			}
		}
	}
}
<commit_msg>Use quietLogger in head credential counter<commit_after>package revok

import (
	"context"
	"cred-alert/db"
	"cred-alert/gitclient"
	"cred-alert/kolsch"
	"cred-alert/scanners"
	"cred-alert/scanners/filescanner"
	"cred-alert/sniff"
	"fmt"
	"io"
	"os"
	"time"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/tedsuo/ifrit"
)

type headCredentialCounter struct {
	logger               lager.Logger
	repositoryRepository db.RepositoryRepository
	clock                clock.Clock
	interval             time.Duration
	gitClient            gitclient.Client
	sniffer              sniff.Sniffer
}

func NewHeadCredentialCounter(
	logger lager.Logger,
	repositoryRepository db.RepositoryRepository,
	clock clock.Clock,
	interval time.Duration,
	gitClient gitclient.Client,
	sniffer sniff.Sniffer,
) ifrit.Runner {
	return &headCredentialCounter{
		logger:               logger,
		repositoryRepository: repositoryRepository,
		clock:                clock,
		interval:             interval,
		gitClient:            gitClient,
		sniffer:              sniffer,
	}
}

func (c *headCredentialCounter) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
	logger := c.logger.Session("head-credential-counter")
	logger.Info("starting")

	close(ready)

	timer := c.clock.NewTicker(c.interval)

	ctx, cancel := context.WithCancel(context.Background())

	c.work(logger, cancel, signals)

	for {
		select {
		case <-timer.C():
			c.work(logger, cancel, signals)
		case <-signals:
			cancel()
			return nil
		case <-ctx.Done():
			return nil
		}
	}
}

func (c *headCredentialCounter) work(
	logger lager.Logger,
	cancel context.CancelFunc,
	signals <-chan os.Signal,
) {
	repositories, err := c.repositoryRepository.All()
	if err != nil {
		logger.Error("failed-getting-all-repositories", err)
	}

	quietLogger := kolsch.NewLogger()

	for i := range repositories {
		select {
		case <-signals:
			cancel()
			return
		default:
			repository := repositories[i]
			repoLogger := logger.WithData(lager.Data{
				"ref":  repository.DefaultBranch,
				"path": repository.Path,
			})

			r, w := io.Pipe()

			errCh := make(chan error)
			go func() {
				errCh <- c.gitClient.AllBlobsForRef(repository.Path, fmt.Sprintf("refs/remotes/origin/%s", repository.DefaultBranch), w)
			}()

			var credCount uint
			_ = c.sniffer.Sniff(
				quietLogger,
				filescanner.New(r, ""), // no filename necessary
				func(lager.Logger, scanners.Violation) error {
					credCount++
					return nil
				})

			if err := <-errCh; err != nil {
				repoLogger.Error("failed-to-get-blobs", err)
				continue
			}

			err := c.repositoryRepository.UpdateCredentialCount(&repository, credCount)
			if err != nil {
				repoLogger.Error("failed-to-update-credential-count", err)
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
   Copyright 2015 Shlomi Noach, courtesy Booking.com

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package process

import (
	"sync"
	"time"

	"github.com/github/orchestrator/go/config"

	"github.com/github/orchestrator/go/raft"
	"github.com/openark/golib/log"
)

const registrationPollSeconds = 5

type NodeHealth struct {
	Hostname        string
	Token           string
	AppVersion      string
	FirstSeenActive string
	LastSeenActive  string
	ExtraInfo       string
	Command         string

	LastReported time.Time
	once         sync.Once
}

func NewNodeHealth() *NodeHealth {
	return &NodeHealth{
		Hostname:   ThisHostname,
		Token:      ProcessToken.Hash,
		AppVersion: config.RuntimeCLIFlags.ConfiguredVersion,
	}
}

var ThisNodeHealth = NewNodeHealth()

type HealthStatus struct {
	Healthy        bool
	Hostname       string
	Token          string
	IsActiveNode   bool
	ActiveNode     NodeHealth
	Error          error
	AvailableNodes [](*NodeHealth)
}

type OrchestratorExecutionMode string

const (
	OrchestratorExecutionCliMode  OrchestratorExecutionMode = "CLIMode"
	OrchestratorExecutionHttpMode                           = "HttpMode"
)

var continuousRegistrationOnce sync.Once

func RegisterNode(nodeHealth *NodeHealth) (healthy bool, err error) {
	nodeHealth.LastReported = time.Now()
	if orcraft.IsRaftEnabled() {
		if orcraft.IsLeader() {
			err := orcraft.PublishCommand("register-node", nodeHealth)
			return err == nil, err
		}
	}
	return WriteRegisterNode(nodeHealth)
}

// HealthTest attempts to write to the backend database and get a result
func HealthTest() (*HealthStatus, error) {
	health := HealthStatus{Healthy: false, Hostname: ThisHostname, Token: ProcessToken.Hash}

	healthy, err := RegisterNode(ThisNodeHealth)
	if err != nil {
		health.Error = err
		return &health, log.Errore(err)
	}
	health.Healthy = healthy
	if orcraft.IsRaftEnabled() {
		health.ActiveNode.Hostname = orcraft.GetLeader()
		health.IsActiveNode = orcraft.IsLeader()
	} else {
		health.ActiveNode, health.IsActiveNode, err = ElectedNode()
		if err != nil {
			health.Error = err
			return &health, log.Errore(err)
		}
	}

	health.AvailableNodes, err = ReadAvailableNodes(true)

	return &health, nil
}

// ContinuousRegistration will continuously update the node_health
// table showing that the current process is still running.
func ContinuousRegistration(extraInfo string, command string) {
	ThisNodeHealth.ExtraInfo = extraInfo
	ThisNodeHealth.Command = command
	continuousRegistrationOnce.Do(func() {
		tickOperation := func() {
			if _, err := RegisterNode(ThisNodeHealth); err != nil {
				log.Errorf("ContinuousRegistration: RegisterNode failed: %+v", err)
			}
		}
		// First one is synchronous
		tickOperation()
		go func() {
			registrationTick := time.Tick(time.Duration(registrationPollSeconds) * time.Second)
			for range registrationTick {
				// We already run inside a go-routine so
				// do not do this asynchronously.  If we
				// get stuck then we don't want to fill up
				// the backend pool with connections running
				// this maintenance operation.
				tickOperation()
			}
		}()
	})
}
<commit_msg>fixed node health hostanme<commit_after>/*
   Copyright 2015 Shlomi Noach, courtesy Booking.com

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package process

import (
	"sync"
	"time"

	"github.com/github/orchestrator/go/config"

	"github.com/github/orchestrator/go/raft"
	"github.com/openark/golib/log"
)

const registrationPollSeconds = 5

type NodeHealth struct {
	Hostname        string
	Token           string
	AppVersion      string
	FirstSeenActive string
	LastSeenActive  string
	ExtraInfo       string
	Command         string

	LastReported time.Time
	once         sync.Once
}

func NewNodeHealth() *NodeHealth {
	return &NodeHealth{
		Hostname:   ThisHostname,
		Token:      ProcessToken.Hash,
		AppVersion: config.RuntimeCLIFlags.ConfiguredVersion,
	}
}

func (nodeHealth *NodeHealth) Update() *NodeHealth {
	nodeHealth.Hostname = ThisHostname
	nodeHealth.LastReported = time.Now()
	return nodeHealth
}

var ThisNodeHealth = NewNodeHealth()

type HealthStatus struct {
	Healthy        bool
	Hostname       string
	Token          string
	IsActiveNode   bool
	ActiveNode     NodeHealth
	Error          error
	AvailableNodes [](*NodeHealth)
}

type OrchestratorExecutionMode string

const (
	OrchestratorExecutionCliMode  OrchestratorExecutionMode = "CLIMode"
	OrchestratorExecutionHttpMode                           = "HttpMode"
)

var continuousRegistrationOnce sync.Once

func RegisterNode(nodeHealth *NodeHealth) (healthy bool, err error) {
	nodeHealth.Update()
	if orcraft.IsRaftEnabled() {
		if orcraft.IsLeader() {
			err := orcraft.PublishCommand("register-node", nodeHealth)
			return err == nil, err
		}
	}
	return WriteRegisterNode(nodeHealth)
}

// HealthTest attempts to write to the backend database and get a result
func HealthTest() (*HealthStatus, error) {
	health := HealthStatus{Healthy: false, Hostname: ThisHostname, Token: ProcessToken.Hash}

	healthy, err := RegisterNode(ThisNodeHealth)
	if err != nil {
		health.Error = err
		return &health, log.Errore(err)
	}
	health.Healthy = healthy
	if orcraft.IsRaftEnabled() {
		health.ActiveNode.Hostname = orcraft.GetLeader()
		health.IsActiveNode = orcraft.IsLeader()
	} else {
		health.ActiveNode, health.IsActiveNode, err = ElectedNode()
		if err != nil {
			health.Error = err
			return &health, log.Errore(err)
		}
	}

	health.AvailableNodes, err = ReadAvailableNodes(true)

	return &health, nil
}

// ContinuousRegistration will continuously update the node_health
// table showing that the current process is still running.
func ContinuousRegistration(extraInfo string, command string) {
	ThisNodeHealth.ExtraInfo = extraInfo
	ThisNodeHealth.Command = command
	continuousRegistrationOnce.Do(func() {
		tickOperation := func() {
			if _, err := RegisterNode(ThisNodeHealth); err != nil {
				log.Errorf("ContinuousRegistration: RegisterNode failed: %+v", err)
			}
		}
		// First one is synchronous
		tickOperation()
		go func() {
			registrationTick := time.Tick(time.Duration(registrationPollSeconds) * time.Second)
			for range registrationTick {
				// We already run inside a go-routine so
				// do not do this asynchronously.  If we
				// get stuck then we don't want to fill up
				// the backend pool with connections running
				// this maintenance operation.
				tickOperation()
			}
		}()
	})
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

func ValidBarcode8(s string) bool {
	re := regexp.MustCompile("^[A-Z0-9]{8,8}$")
	return re.MatchString(s)
}

func ValidBarcode7(s string) bool {
	re := regexp.MustCompile("^[A-Z0-9]{7,7}$")
	return re.MatchString(s)
}

func ValidCarton20(s string) bool {
	re := regexp.MustCompile("^[0-9]{20,20}$")
	return re.MatchString(s)
}

func ReverseProxy(proxyPort string, pathMap map[string]string) {

	for urlPath, targetPort := range pathMap {
		u, _ := url.Parse("http://127.0.0.1:" + targetPort)
		http.Handle(urlPath, httputil.NewSingleHostReverseProxy(u))
	}

	http.ListenAndServe(":"+proxyPort, nil)

}

func TimeNowString() string {
	return fmt.Sprintf("%v", time.Unix(0, time.Now().UnixNano()/(int64(time.Millisecond)/int64(time.Nanosecond))*int64(time.Millisecond)))[:23]
}

func EscapeLatex(s string) string {
	s2 := strings.Replace(s, "\\", "\\textbackslash", -1)
	s2 = strings.Replace(s2, "&", "\\&", -1)
	s2 = strings.Replace(s2, "%", "\\%", -1)
	s2 = strings.Replace(s2, "$", "\\$", -1)
	s2 = strings.Replace(s2, "#", "\\#", -1)
	s2 = strings.Replace(s2, "_", "\\_", -1)
	s2 = strings.Replace(s2, "{", "\\{", -1)
	s2 = strings.Replace(s2, "}", "\\}", -1)
	s2 = strings.Replace(s2, "~", "\\textasciitilde", -1)
	return strings.Replace(s2, "^", "\\textasciicircum", -1)

}

func PadZero(s string) string {
	if s[0] == '.' {
		return "0" + s
	} else {
		return s
	}
}

type TmpRow7 struct {
	Str1   string
	Str2   string
	Str3   string
	Str4   string
	Str5   string
	Str6   string
	Str7   string
	Int1   int64
	Int2   int64
	Int3   int64
	Int4   int64
	Int5   int64
	Int6   int64
	Int7   int64
	Float1 float64
	Float2 float64
	Float3 float64
	Float4 float64
	Float5 float64
	Float6 float64
	Float7 float64
}

func IndexOf(target int, intSlice []int) int {
	for i, v := range intSlice {
		if v == target {
			return i
		}
	}
	return -1
}

func StringToRows7(data string, config []string) []TmpRow7 {
	var translate [21]int
	strPosition := 1
	intPosition := 8
	floatPosition := 15
	var result []TmpRow7

	for j, t := range config {
		if t == "string" {
			translate[j] = strPosition
			strPosition++
		} else if t == "int64" {
			translate[j] = intPosition
			intPosition++

		} else if t == "float64" {
			translate[j] = floatPosition
			floatPosition++
		}

	}

	lines := strings.Split(data, "\n")
	for _, v := range lines {
		if v == "" {
			continue
		}
		columns := strings.Split(v, "\t")
		row := TmpRow7{}
		index := 0
		for j, col := range columns {
			index = translate[j]
			switch {
			case index == 1:
				row.Str1 = col
			case index == 2:
				row.Str2 = col
			case index == 3:
				row.Str3 = col
			case index == 4:
				row.Str4 = col
			case index == 5:
				row.Str5 = col
			case index == 6:
				row.Str6 = col
			case index == 7:
				row.Str7 = col
			case index >= 7 && index <= 14:
				tmpInt, err := strconv.ParseInt(col, 10, 64)
				if err != nil {
					tmpInt = 0
				}
				switch {
				case index == 8:
					row.Int1 = tmpInt
				case index == 9:
					row.Int2 = tmpInt
				case index == 10:
					row.Int3 = tmpInt
				case index == 11:
					row.Int4 = tmpInt
				case index == 12:
					row.Int5 = tmpInt
				case index == 13:
					row.Int6 = tmpInt
				case index == 14:
					row.Int7 = tmpInt
				}
			case index >= 15 && index <= 21:
				tmpFloat, err := strconv.ParseFloat(col, 64)
				if err != nil {
					tmpFloat = 0
				}
				switch {
				case index == 15:
					row.Float1 = tmpFloat
				case index == 16:
					row.Float2 = tmpFloat
				case index == 17:
					row.Float3 = tmpFloat
				case index == 18:
					row.Float4 = tmpFloat
				case index == 19:
					row.Float5 = tmpFloat
				case index == 20:
					row.Float6 = tmpFloat
				case index == 21:
					row.Float7 = tmpFloat
				}

			}
		}

		result = append(result, row)
	}
	return result
}
<commit_msg>=added base64<commit_after>package common

import (
	"encoding/base64"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

func ValidBarcode8(s string) bool {
	re := regexp.MustCompile("^[A-Z0-9]{8,8}$")
	return re.MatchString(s)
}

func ValidBarcode7(s string) bool {
	re := regexp.MustCompile("^[A-Z0-9]{7,7}$")
	return re.MatchString(s)
}

func ValidCarton20(s string) bool {
	re := regexp.MustCompile("^[0-9]{20,20}$")
	return re.MatchString(s)
}

func ReverseProxy(proxyPort string, pathMap map[string]string) {

	for urlPath, targetPort := range pathMap {
		u, _ := url.Parse("http://127.0.0.1:" + targetPort)
		http.Handle(urlPath, httputil.NewSingleHostReverseProxy(u))
	}

	http.ListenAndServe(":"+proxyPort, nil)

}

func TimeNowString() string {
	return fmt.Sprintf("%v", time.Unix(0, time.Now().UnixNano()/(int64(time.Millisecond)/int64(time.Nanosecond))*int64(time.Millisecond)))[:23]
}

func EscapeLatex(s string) string {
	s2 := strings.Replace(s, "\\", "\\textbackslash", -1)
	s2 = strings.Replace(s2, "&", "\\&", -1)
	s2 = strings.Replace(s2, "%", "\\%", -1)
	s2 = strings.Replace(s2, "$", "\\$", -1)
	s2 = strings.Replace(s2, "#", "\\#", -1)
	s2 = strings.Replace(s2, "_", "\\_", -1)
	s2 = strings.Replace(s2, "{", "\\{", -1)
	s2 = strings.Replace(s2, "}", "\\}", -1)
	s2 = strings.Replace(s2, "~", "\\textasciitilde", -1)
	return strings.Replace(s2, "^", "\\textasciicircum", -1)

}

func PadZero(s string) string {
	if s[0] == '.' {
		return "0" + s
	} else {
		return s
	}
}

type TmpRow7 struct {
	Str1   string
	Str2   string
	Str3   string
	Str4   string
	Str5   string
	Str6   string
	Str7   string
	Int1   int64
	Int2   int64
	Int3   int64
	Int4   int64
	Int5   int64
	Int6   int64
	Int7   int64
	Float1 float64
	Float2 float64
	Float3 float64
	Float4 float64
	Float5 float64
	Float6 float64
	Float7 float64
}

func IndexOf(target int, intSlice []int) int {
	for i, v := range intSlice {
		if v == target {
			return i
		}
	}
	return -1
}

func StringToRows7(data string, config []string) []TmpRow7 {
	var translate [21]int
	strPosition := 1
	intPosition := 8
	floatPosition := 15
	var result []TmpRow7

	for j, t := range config {
		if t == "string" {
			translate[j] = strPosition
			strPosition++
		} else if t == "int64" {
			translate[j] = intPosition
			intPosition++

		} else if t == "float64" {
			translate[j] = floatPosition
			floatPosition++
		}

	}

	lines := strings.Split(data, "\n")
	for _, v := range lines {
		if v == "" {
			continue
		}
		columns := strings.Split(v, "\t")
		row := TmpRow7{}
		index := 0
		for j, col := range columns {
			index = translate[j]
			switch {
			case index == 1:
				row.Str1 = col
			case index == 2:
				row.Str2 = col
			case index == 3:
				row.Str3 = col
			case index == 4:
				row.Str4 = col
			case index == 5:
				row.Str5 = col
			case index == 6:
				row.Str6 = col
			case index == 7:
				row.Str7 = col
			case index >= 7 && index <= 14:
				tmpInt, err := strconv.ParseInt(col, 10, 64)
				if err != nil {
					tmpInt = 0
				}
				switch {
				case index == 8:
					row.Int1 = tmpInt
				case index == 9:
					row.Int2 = tmpInt
				case index == 10:
					row.Int3 = tmpInt
				case index == 11:
					row.Int4 = tmpInt
				case index == 12:
					row.Int5 = tmpInt
				case index == 13:
					row.Int6 = tmpInt
				case index == 14:
					row.Int7 = tmpInt
				}
			case index >= 15 && index <= 21:
				tmpFloat, err := strconv.ParseFloat(col, 64)
				if err != nil {
					tmpFloat = 0
				}
				switch {
				case index == 15:
					row.Float1 = tmpFloat
				case index == 16:
					row.Float2 = tmpFloat
				case index == 17:
					row.Float3 = tmpFloat
				case index == 18:
					row.Float4 = tmpFloat
				case index == 19:
					row.Float5 = tmpFloat
				case index == 20:
					row.Float6 = tmpFloat
				case index == 21:
					row.Float7 = tmpFloat
				}

			}
		}

		result = append(result, row)
	}
	return result
}

func Base64Encode(msg string) string {
	return base64.StdEncoding.EncodeToString([]byte(msg))
}

func Base64Decode(encoded string) string {
	decoded, err := base64.StdEncoding.DecodeString(encoded)
	if err != nil {
		return ""
	}
	return string(decoded)

}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os/exec"
	"strings"
	"time"

	"github.com/brotherlogic/goserver/utils"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pbb "github.com/brotherlogic/buildserver/proto"
	pbfc "github.com/brotherlogic/filecopier/proto"
	pb "github.com/brotherlogic/gobuildslave/proto"
	pbvt "github.com/brotherlogic/versiontracker/proto"
)

const (
	pendWait = time.Minute
)

var (
	ackQueueLen = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "gobuildslave_ackqueuelen",
		Help: "The size of the ack queue",
	})
)

func (s *Server) procAcks() {
	for job := range s.ackChan {
		ackQueueLen.Set(float64(len(s.ackChan)))
		ctx, cancel := utils.ManualContext("gobuildslaveack", time.Minute)
		conn, err := s.FDialSpecificServer(ctx, "versiontracker", s.Registry.GetIdentifier())
		if err != nil {
			s.DLog(ctx, fmt.Sprintf("Dial error: (%v), %v\n", job.GetJob(), err))
			s.ackChan <- job
		} else {

			client := pbvt.NewVersionTrackerServiceClient(conn)
			_, err = client.NewJob(ctx, &pbvt.NewJobRequest{Version: &pbb.Version{Version: job.GetRunningVersion(), Job: job.GetJob()}})
			if err != nil {
				s.ackChan <- job
			}
			conn.Close()
		}
		cancel()

		// Don't rush the system
		time.Sleep(time.Second)
	}
}

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	s.DLog(ctx, fmt.Sprintf("TRANS: %v\n", job))
	startState := job.State
	job.LastUpdateTime = time.Now().Unix()
	switch job.State {
	case pb.State_WARMUP:
		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		job.RunningVersion = elems[0]
		s.ackChan <- job
		job.State = pb.State_ACKNOWLEDGED
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job)
		job.SubState = fmt.Sprintf("SCHED: %v @ %v", key, time.Now())
		if !job.Job.Bootstrap {
			if key != "" {
				job.State = pb.State_BUILT
				job.RunningVersion = key
				s.ackChan <- job
			} else {
				// Bootstrap this job since we don't have an initial version
				if job.Job.PartialBootstrap {
					job.Job.Bootstrap = true
				}
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
		}
	case pb.State_BUILDING:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILD(%v): %v", job.CommandKey, s.scheduler.getState(job.CommandKey))
		s.stateMutex.Unlock()
		s.scheduler.wait(job.CommandKey)
		job.State = pb.State_BUILT
	case pb.State_BUILT:
		job.SubState = "Getting Output"
		output, _ := s.scheduler.getOutput(job.CommandKey)
		job.SubState = "Entering Lock"
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILT(%v): (%v): %v", job.CommandKey, len(output), output)
		s.stateMutex.Unlock()
		if job.Job.Bootstrap && len(output) > 0 {
			if job.BuildFail == 5 {
				job.SubState = "Sending Report"
				s.deliverCrashReport(ctx, job, output)
				job.BuildFail = 0
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.State = pb.State_VERSION_CHECK
			// Don't check version on pb jobs
			if job.GetJob().GetPartialBootstrap() {
				s.doCopy(job)
				job.BuildFail = 0
				job.SubState = "Scheduling A Run post copy"
				key := s.scheduleRun(job)
				job.CommandKey = key
				job.StartTime = time.Now().Unix()
				job.State = pb.State_PENDING
				if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
					s.pendingMap[time.Now().Weekday()] = make(map[string]int)
				}
				s.pendingMap[time.Now().Weekday()][job.Job.Name]++
			}
		}
		job.SubState = "Out of case"
	case pb.State_VERSION_CHECK:
		s.doCopy(job)
		s.loadCurrentVersions()
		version, err := s.getLatestVersion(ctx, job.GetJob().Name, job.GetJob().GetGoPath())
		if err != nil {
			s.Log(fmt.Sprintf("Error getting version: %v", err))
			break
		}

		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		job.RunningVersion = elems[0]
		s.ackChan <- job

		s.versionsMutex.Lock()
		defer s.versionsMutex.Unlock()
		if elems[0] != version.GetVersion() {
			s.Log(fmt.Sprintf("Bad version on %v for %v -> %v vs %v", s.Registry.Identifier, job.GetJob().GetName(), elems[0], s.versions[job.GetJob().GetName()].Version))
			job.SubState = fmt.Sprintf("Dealing With Version Mismatch: %v", job.BuildFail)

			conn, err := s.FDialSpecificServer(ctx, "versiontracker", s.Registry.Identifier)
			if err != nil {
				s.Log(fmt.Sprintf("Unable to dial vt: %v", err))
			} else {
				defer conn.Close()
				vtc := pbvt.NewVersionTrackerServiceClient(conn)
				_, err = vtc.NewVersion(ctx, &pbvt.NewVersionRequest{
					Version: version,
				})
				s.Log(fmt.Sprintf("Requested new version: %v", err))
			}

			// Don't let the job sit here
			job.BuildFail++
			if job.BuildFail > 10 {
				job.State = pb.State_ACKNOWLEDGED

			}
			break
		}

		job.BuildFail = 0
		job.SubState = "Scheduling of the Run"
		key := s.scheduleRun(job)
		job.CommandKey = key
		job.StartTime = time.Now().Unix()
		job.State = pb.State_PENDING
		if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
			s.pendingMap[time.Now().Weekday()] = make(map[string]int)
		}
		s.pendingMap[time.Now().Weekday()][job.Job.Name]++

	case pb.State_PENDING:
		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		job.RunningVersion = elems[0]
		s.ackChan <- job

		if job.Job.PartialBootstrap && job.Job.Bootstrap {
			job.Job.Bootstrap = false
		}
		s.stateMutex.Lock()
		out, _ := s.scheduler.getOutput(job.CommandKey)
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", out)
		s.stateMutex.Unlock()
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			var err error
			if job.Job.Name == "discovery" {
				err = s.runOnChange()
			}
			code := status.Convert(err).Code()

			//Unavailable allows the job to die here
			if code == codes.OK || code == codes.Unavailable {
				// Validate that the job is alive
				if !s.isJobAlive(ctx, job) {
					s.Log(fmt.Sprintf("Job %v is not alive", job))
				}
				job.State = pb.State_RUNNING
			} else {
				s.Log(fmt.Sprintf("Cannot reregister: %v", err))
			}
		}
	case pb.State_RUNNING:
		output, errout := s.scheduler.getOutput(job.CommandKey)
		output2, _ := s.scheduler.getErrOutput(job.CommandKey)
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("ROUTPUT = %v, %v", output, s.scheduler.getStatus(job.CommandKey))
		job.Status = s.scheduler.getStatus(job.CommandKey)
		s.stateMutex.Unlock()
		if len(job.CommandKey) > 0 {
			s.scheduler.wait(job.CommandKey)
			s.stateMap[job.Job.Name] = fmt.Sprintf("ONLOCk = (%v, %v)", job, output)
			s.stateMutex.Lock()
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = (%v, %v)", job, output)
			s.stateMutex.Unlock()
			s.deliverCrashReport(ctx, job, fmt.Sprintf("%v%v", output, output2))
			job.State = pb.State_DIED
		}

		if s.Registry != nil {
			entry, err := s.FFindSpecificServer(ctx, job.Job.Name, s.Registry.Identifier)
			if err != nil {
				if job.DiscoverCount > 30 {
					output2, errout2 := s.scheduler.getErrOutput(job.CommandKey)
					s.RaiseIssue("Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running (%v) the output says %v (%v), %v, %v", job.Job.Name, s.Registry.Identifier, err, output, errout, output2, errout2))
				}
				job.DiscoverCount++
				s.Log(fmt.Sprintf("Missing discover for %+v, %v -> %v", job, entry, err))
			} else {
				job.Port = entry.GetPort()
				job.DiscoverCount = 0
			}
		}

		// Restart this job if we need to
		if !job.Job.Bootstrap {
			if time.Now().Sub(time.Unix(job.LastVersionPull, 0)) > time.Minute*5 {
				version, err := s.getVersion(ctx, job.Job)
				job.LastVersionPull = time.Now().Unix()

				if err == nil && version.Version != job.RunningVersion {
					s.stateMutex.Lock()
					s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
					s.stateMutex.Unlock()
					s.scheduler.killJob(job.CommandKey)
				}
			}
		}
	case pb.State_BRINK_OF_DEATH:
		if s.version.confirm(ctx, job.Job.Name) {
			s.scheduler.killJob(job.CommandKey)
			job.State = pb.State_ACKNOWLEDGED
		}
	case pb.State_DIED:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("DIED %v", job.CommandKey)
		s.stateMutex.Unlock()
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != startState {
		job.LastTransitionTime = time.Now().Unix()
	}
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) (*pbb.Version, error) {
	version, err := s.builder.build(ctx, job)
	if err != nil {
		return &pbb.Version{}, err
	}

	return version, nil
}

func updateJob(err error, job *pb.JobAssignment, resp *pbfc.CopyResponse) {
	if err == nil {
		job.QueuePos = resp.IndexInQueue
	}

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.JobAssignment) string {
	if job.Job.Bootstrap {
		c := s.translator.build(job.Job)
		// Block builds to prevent clashes
		return s.scheduler.Schedule(&rCommand{command: c, base: job.Job.Name, block: true})
	}

	val, err := s.builder.build(ctx, job.Job)
	if err != nil {
		s.DLog(ctx, fmt.Sprintf("BUILD Error: %v\n", err))
		return ""
	}
	return val.Version
}

func (s *Server) doCopy(job *pb.JobAssignment) {
	//Copy over any existing new versions

	key := s.scheduler.Schedule(&rCommand{command: exec.Command("mv", "$GOPATH/bin/"+job.GetJob().GetName()+".new", "$GOPATH/bin/"+job.GetJob().GetName()), base: job.GetJob().GetName()})
	s.scheduler.wait(key)

	key = s.scheduler.Schedule(&rCommand{command: exec.Command("mv", "$GOPATH/bin/"+job.GetJob().GetName()+".nversion", "$GOPATH/bin/"+job.GetJob().GetName()+".version"), base: job.GetJob().GetName() + ".version"})
	s.scheduler.wait(key)
}

func (s *Server) scheduleRun(job *pb.JobAssignment) string {
	// Wait a while before starting the job JIC
	time.Sleep(time.Second * 2)

	c := s.translator.run(job.GetJob())
	return s.scheduler.Schedule(&rCommand{command: c, base: job.GetJob().GetName()})
}
<commit_msg>Better ack handling<commit_after>package main

import (
	"context"
	"fmt"
	"os/exec"
	"strings"
	"time"

	"github.com/brotherlogic/goserver/utils"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pbb "github.com/brotherlogic/buildserver/proto"
	pbfc "github.com/brotherlogic/filecopier/proto"
	pb "github.com/brotherlogic/gobuildslave/proto"
	pbvt "github.com/brotherlogic/versiontracker/proto"
)

const (
	pendWait = time.Minute
)

var (
	ackQueueLen = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "gobuildslave_ackqueuelen",
		Help: "The size of the ack queue",
	})
)

func (s *Server) procAcks() {
	for job := range s.ackChan {
		done := false
		for !done {
			ackQueueLen.Set(float64(len(s.ackChan)))
			ctx, cancel := utils.ManualContext("gobuildslaveack", time.Minute)
			conn, err := s.FDialSpecificServer(ctx, "versiontracker", s.Registry.GetIdentifier())
			if err != nil {
				s.DLog(ctx, fmt.Sprintf("Dial error: (%v), %v\n", job.GetJob(), err))
			} else {
				client := pbvt.NewVersionTrackerServiceClient(conn)
				_, err = client.NewJob(ctx, &pbvt.NewJobRequest{Version: &pbb.Version{Version: job.GetRunningVersion(), Job: job.GetJob()}})
				if err == nil {
					done = true
				}
				conn.Close()
			}
			cancel()
		}

		// Don't rush the system
		time.Sleep(time.Second)
	}
}

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	s.DLog(ctx, fmt.Sprintf("TRANS: %v\n", job))
	startState := job.State
	job.LastUpdateTime = time.Now().Unix()
	switch job.State {
	case pb.State_WARMUP:
		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		job.RunningVersion = elems[0]
		s.ackChan <- job
		job.State = pb.State_ACKNOWLEDGED
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job)
		job.SubState = fmt.Sprintf("SCHED: %v @ %v", key, time.Now())
		if !job.Job.Bootstrap {
			if key != "" {
				job.State = pb.State_BUILT
				job.RunningVersion = key
				s.ackChan <- job
			} else {
				// Bootstrap this job since we don't have an initial version
				if job.Job.PartialBootstrap {
					job.Job.Bootstrap = true
				}
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
		}
	case pb.State_BUILDING:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILD(%v): %v", job.CommandKey, s.scheduler.getState(job.CommandKey))
		s.stateMutex.Unlock()
		s.scheduler.wait(job.CommandKey)
		job.State = pb.State_BUILT
	case pb.State_BUILT:
		job.SubState = "Getting Output"
		output, _ := s.scheduler.getOutput(job.CommandKey)
		job.SubState = "Entering Lock"
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILT(%v): (%v): %v", job.CommandKey, len(output), output)
		s.stateMutex.Unlock()
		if job.Job.Bootstrap && len(output) > 0 {
			if job.BuildFail == 5 {
				job.SubState = "Sending Report"
				s.deliverCrashReport(ctx, job, output)
				job.BuildFail = 0
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.State = pb.State_VERSION_CHECK
			// Don't check version on pb jobs
			if job.GetJob().GetPartialBootstrap() {
				s.doCopy(job)
				job.BuildFail = 0
				job.SubState = "Scheduling A Run post copy"
				key := s.scheduleRun(job)
				job.CommandKey = key
				job.StartTime = time.Now().Unix()
				job.State = pb.State_PENDING
				if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
					s.pendingMap[time.Now().Weekday()] = make(map[string]int)
				}
				s.pendingMap[time.Now().Weekday()][job.Job.Name]++
			}
		}
		job.SubState = "Out of case"
	case pb.State_VERSION_CHECK:
		s.doCopy(job)
		s.loadCurrentVersions()
		version, err := s.getLatestVersion(ctx, job.GetJob().Name, job.GetJob().GetGoPath())
		if err != nil {
			s.Log(fmt.Sprintf("Error getting version: %v", err))
			break
		}

		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		job.RunningVersion = elems[0]
		s.ackChan <- job

		s.versionsMutex.Lock()
		defer s.versionsMutex.Unlock()
		if elems[0] != version.GetVersion() {
			s.Log(fmt.Sprintf("Bad version on %v for %v -> %v vs %v", s.Registry.Identifier, job.GetJob().GetName(), elems[0], s.versions[job.GetJob().GetName()].Version))
			job.SubState = fmt.Sprintf("Dealing With Version Mismatch: %v", job.BuildFail)

			conn, err := s.FDialSpecificServer(ctx, "versiontracker", s.Registry.Identifier)
			if err != nil {
				s.Log(fmt.Sprintf("Unable to dial vt: %v", err))
			} else {
				defer conn.Close()
				vtc := pbvt.NewVersionTrackerServiceClient(conn)
				_, err = vtc.NewVersion(ctx, &pbvt.NewVersionRequest{
					Version: version,
				})
				s.Log(fmt.Sprintf("Requested new version: %v", err))
			}

			// Don't let the job sit here
			job.BuildFail++
			if job.BuildFail > 10 {
				job.State = pb.State_ACKNOWLEDGED

			}
			break
		}

		job.BuildFail = 0
		job.SubState = "Scheduling of the Run"
		key := s.scheduleRun(job)
		job.CommandKey = key
		job.StartTime = time.Now().Unix()
		job.State = pb.State_PENDING
		if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
			s.pendingMap[time.Now().Weekday()] = make(map[string]int)
		}
		s.pendingMap[time.Now().Weekday()][job.Job.Name]++

	case pb.State_PENDING:
		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		job.RunningVersion = elems[0]
		s.ackChan <- job

		if job.Job.PartialBootstrap && job.Job.Bootstrap {
			job.Job.Bootstrap = false
		}
		s.stateMutex.Lock()
		out, _ := s.scheduler.getOutput(job.CommandKey)
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", out)
		s.stateMutex.Unlock()
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			var err error
			if job.Job.Name == "discovery" {
				err = s.runOnChange()
			}
			code := status.Convert(err).Code()

			//Unavailable allows the job to die here
			if code == codes.OK || code == codes.Unavailable {
				// Validate that the job is alive
				if !s.isJobAlive(ctx, job) {
					s.Log(fmt.Sprintf("Job %v is not alive", job))
				}
				job.State = pb.State_RUNNING
			} else {
				s.Log(fmt.Sprintf("Cannot reregister: %v", err))
			}
		}
	case pb.State_RUNNING:
		output, errout := s.scheduler.getOutput(job.CommandKey)
		output2, _ := s.scheduler.getErrOutput(job.CommandKey)
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("ROUTPUT = %v, %v", output, s.scheduler.getStatus(job.CommandKey))
		job.Status = s.scheduler.getStatus(job.CommandKey)
		s.stateMutex.Unlock()
		if len(job.CommandKey) > 0 {
			s.scheduler.wait(job.CommandKey)
			s.stateMap[job.Job.Name] = fmt.Sprintf("ONLOCk = (%v, %v)", job, output)
			s.stateMutex.Lock()
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = (%v, %v)", job, output)
			s.stateMutex.Unlock()
			s.deliverCrashReport(ctx, job, fmt.Sprintf("%v%v", output, output2))
			job.State = pb.State_DIED
		}

		if s.Registry != nil {
			entry, err := s.FFindSpecificServer(ctx, job.Job.Name, s.Registry.Identifier)
			if err != nil {
				if job.DiscoverCount > 30 {
					output2, errout2 := s.scheduler.getErrOutput(job.CommandKey)
					s.RaiseIssue("Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running (%v) the output says %v (%v), %v, %v", job.Job.Name, s.Registry.Identifier, err, output, errout, output2, errout2))
				}
				job.DiscoverCount++
				s.Log(fmt.Sprintf("Missing discover for %+v, %v -> %v", job, entry, err))
			} else {
				job.Port = entry.GetPort()
				job.DiscoverCount = 0
			}
		}

		// Restart this job if we need to
		if !job.Job.Bootstrap {
			if time.Now().Sub(time.Unix(job.LastVersionPull, 0)) > time.Minute*5 {
				version, err := s.getVersion(ctx, job.Job)
				job.LastVersionPull = time.Now().Unix()

				if err == nil && version.Version != job.RunningVersion {
					s.stateMutex.Lock()
					s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
					s.stateMutex.Unlock()
					s.scheduler.killJob(job.CommandKey)
				}
			}
		}
	case pb.State_BRINK_OF_DEATH:
		if s.version.confirm(ctx, job.Job.Name) {
			s.scheduler.killJob(job.CommandKey)
			job.State = pb.State_ACKNOWLEDGED
		}
	case pb.State_DIED:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("DIED %v", job.CommandKey)
		s.stateMutex.Unlock()
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != startState {
		job.LastTransitionTime = time.Now().Unix()
	}
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) (*pbb.Version, error) {
	version, err := s.builder.build(ctx, job)
	if err != nil {
		return &pbb.Version{}, err
	}

	return version, nil
}

func updateJob(err error, job *pb.JobAssignment, resp *pbfc.CopyResponse) {
	if err == nil {
		job.QueuePos = resp.IndexInQueue
	}

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.JobAssignment) string {
	if job.Job.Bootstrap {
		c := s.translator.build(job.Job)
		// Block builds to prevent clashes
		return s.scheduler.Schedule(&rCommand{command: c, base: job.Job.Name, block: true})
	}

	val, err := s.builder.build(ctx, job.Job)
	if err != nil {
		s.DLog(ctx, fmt.Sprintf("BUILD Error: %v\n", err))
		return ""
	}
	return val.Version
}

func (s *Server) doCopy(job *pb.JobAssignment) {
	//Copy over any existing new versions

	key := s.scheduler.Schedule(&rCommand{command: exec.Command("mv", "$GOPATH/bin/"+job.GetJob().GetName()+".new", "$GOPATH/bin/"+job.GetJob().GetName()), base: job.GetJob().GetName()})
	s.scheduler.wait(key)

	key = s.scheduler.Schedule(&rCommand{command: exec.Command("mv", "$GOPATH/bin/"+job.GetJob().GetName()+".nversion", "$GOPATH/bin/"+job.GetJob().GetName()+".version"), base: job.GetJob().GetName() + ".version"})
	s.scheduler.wait(key)
}

func (s *Server) scheduleRun(job *pb.JobAssignment) string {
	// Wait a while before starting the job JIC
	time.Sleep(time.Second * 2)

	c := s.translator.run(job.GetJob())
	return s.scheduler.Schedule(&rCommand{command: c, base: job.GetJob().GetName()})
}
<|endoftext|>


<commit_before>package common

import (
	"encoding/base64"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

func ValidBarcode8(s string) bool {
	re := regexp.MustCompile("^[A-Z0-9]{8,8}$")
	return re.MatchString(s)
}

func ValidBarcode7(s string) bool {
	re := regexp.MustCompile("^[A-Z0-9]{7,7}$")
	return re.MatchString(s)
}

func ValidCarton20(s string) bool {
	re := regexp.MustCompile("^[0-9]{20,20}$")
	return re.MatchString(s)
}

func ReverseProxy(proxyPort string, pathMap map[string]string) {

	for urlPath, targetPort := range pathMap {
		u, _ := url.Parse("http://127.0.0.1:" + targetPort)
		http.Handle(urlPath, httputil.NewSingleHostReverseProxy(u))
	}

	http.ListenAndServe(":"+proxyPort, nil)

}

func TimeNowString() string {
	return fmt.Sprintf("%v", time.Unix(0, time.Now().UnixNano()/(int64(time.Millisecond)/int64(time.Nanosecond))*int64(time.Millisecond)))[:23]
}

func EscapeLatex(s string) string {
	s2 := strings.Replace(s, "\\", "\\textbackslash", -1)
	s2 = strings.Replace(s2, "&", "\\&", -1)
	s2 = strings.Replace(s2, "%", "\\%", -1)
	s2 = strings.Replace(s2, "$", "\\$", -1)
	s2 = strings.Replace(s2, "#", "\\#", -1)
	s2 = strings.Replace(s2, "_", "\\_", -1)
	s2 = strings.Replace(s2, "{", "\\{", -1)
	s2 = strings.Replace(s2, "}", "\\}", -1)
	s2 = strings.Replace(s2, "~", "\\textasciitilde", -1)
	return strings.Replace(s2, "^", "\\textasciicircum", -1)

}

func PadZero(s string) string {
	if s[0] == '.' {
		return "0" + s
	} else {
		return s
	}
}

type TmpRow7 struct {
	Str1   string
	Str2   string
	Str3   string
	Str4   string
	Str5   string
	Str6   string
	Str7   string
	Int1   int64
	Int2   int64
	Int3   int64
	Int4   int64
	Int5   int64
	Int6   int64
	Int7   int64
	Float1 float64
	Float2 float64
	Float3 float64
	Float4 float64
	Float5 float64
	Float6 float64
	Float7 float64
}

func IndexOf(target int, intSlice []int) int {
	for i, v := range intSlice {
		if v == target {
			return i
		}
	}
	return -1
}

func StringToRows7(data string, config []string) []TmpRow7 {
	var translate [21]int
	strPosition := 1
	intPosition := 8
	floatPosition := 15
	var result []TmpRow7

	for j, t := range config {
		if t == "string" {
			translate[j] = strPosition
			strPosition++
		} else if t == "int64" {
			translate[j] = intPosition
			intPosition++

		} else if t == "float64" {
			translate[j] = floatPosition
			floatPosition++
		}

	}

	lines := strings.Split(data, "\n")
	for _, v := range lines {
		if v == "" {
			continue
		}
		columns := strings.Split(v, "\t")
		row := TmpRow7{}
		index := 0
		for j, col := range columns {
			index = translate[j]
			switch {
			case index == 1:
				row.Str1 = col
			case index == 2:
				row.Str2 = col
			case index == 3:
				row.Str3 = col
			case index == 4:
				row.Str4 = col
			case index == 5:
				row.Str5 = col
			case index == 6:
				row.Str6 = col
			case index == 7:
				row.Str7 = col
			case index >= 7 && index <= 14:
				tmpInt, err := strconv.ParseInt(col, 10, 64)
				if err != nil {
					tmpInt = 0
				}
				switch {
				case index == 8:
					row.Int1 = tmpInt
				case index == 9:
					row.Int2 = tmpInt
				case index == 10:
					row.Int3 = tmpInt
				case index == 11:
					row.Int4 = tmpInt
				case index == 12:
					row.Int5 = tmpInt
				case index == 13:
					row.Int6 = tmpInt
				case index == 14:
					row.Int7 = tmpInt
				}
			case index >= 15 && index <= 21:
				tmpFloat, err := strconv.ParseFloat(col, 64)
				if err != nil {
					tmpFloat = 0
				}
				switch {
				case index == 15:
					row.Float1 = tmpFloat
				case index == 16:
					row.Float2 = tmpFloat
				case index == 17:
					row.Float3 = tmpFloat
				case index == 18:
					row.Float4 = tmpFloat
				case index == 19:
					row.Float5 = tmpFloat
				case index == 20:
					row.Float6 = tmpFloat
				case index == 21:
					row.Float7 = tmpFloat
				}

			}
		}

		result = append(result, row)
	}
	return result
}

func Base64Encode(msg string) string {
	return base64.StdEncoding.EncodeToString([]byte(msg))
}

func Base64Decode(encoded string) string {
	decoded, err := base64.StdEncoding.DecodeString(encoded)
	if err != nil {
		return ""
	}
	return string(decoded)

}
<commit_msg>fix TimeNowString<commit_after>package common

import (
	"encoding/base64"
	"net/http"
	"net/http/httputil"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

func ValidBarcode8(s string) bool {
	re := regexp.MustCompile("^[A-Z0-9]{8,8}$")
	return re.MatchString(s)
}

func ValidBarcode7(s string) bool {
	re := regexp.MustCompile("^[A-Z0-9]{7,7}$")
	return re.MatchString(s)
}

func ValidCarton20(s string) bool {
	re := regexp.MustCompile("^[0-9]{20,20}$")
	return re.MatchString(s)
}

func ReverseProxy(proxyPort string, pathMap map[string]string) {

	for urlPath, targetPort := range pathMap {
		u, _ := url.Parse("http://127.0.0.1:" + targetPort)
		http.Handle(urlPath, httputil.NewSingleHostReverseProxy(u))
	}

	http.ListenAndServe(":"+proxyPort, nil)

}

func TimeNowString() string {
	//return fmt.Sprintf("%v", time.Unix(0, time.Now().UnixNano()/(int64(time.Millisecond)/int64(time.Nanosecond))*int64(time.Millisecond)))[:23]
	return time.Now().Format("2006-01-02 15:04:05.000")
}

func EscapeLatex(s string) string {
	s2 := strings.Replace(s, "\\", "\\textbackslash", -1)
	s2 = strings.Replace(s2, "&", "\\&", -1)
	s2 = strings.Replace(s2, "%", "\\%", -1)
	s2 = strings.Replace(s2, "$", "\\$", -1)
	s2 = strings.Replace(s2, "#", "\\#", -1)
	s2 = strings.Replace(s2, "_", "\\_", -1)
	s2 = strings.Replace(s2, "{", "\\{", -1)
	s2 = strings.Replace(s2, "}", "\\}", -1)
	s2 = strings.Replace(s2, "~", "\\textasciitilde", -1)
	return strings.Replace(s2, "^", "\\textasciicircum", -1)

}

func PadZero(s string) string {
	if s[0] == '.' {
		return "0" + s
	} else {
		return s
	}
}

type TmpRow7 struct {
	Str1   string
	Str2   string
	Str3   string
	Str4   string
	Str5   string
	Str6   string
	Str7   string
	Int1   int64
	Int2   int64
	Int3   int64
	Int4   int64
	Int5   int64
	Int6   int64
	Int7   int64
	Float1 float64
	Float2 float64
	Float3 float64
	Float4 float64
	Float5 float64
	Float6 float64
	Float7 float64
}

func IndexOf(target int, intSlice []int) int {
	for i, v := range intSlice {
		if v == target {
			return i
		}
	}
	return -1
}

func StringToRows7(data string, config []string) []TmpRow7 {
	var translate [21]int
	strPosition := 1
	intPosition := 8
	floatPosition := 15
	var result []TmpRow7

	for j, t := range config {
		if t == "string" {
			translate[j] = strPosition
			strPosition++
		} else if t == "int64" {
			translate[j] = intPosition
			intPosition++

		} else if t == "float64" {
			translate[j] = floatPosition
			floatPosition++
		}

	}

	lines := strings.Split(data, "\n")
	for _, v := range lines {
		if v == "" {
			continue
		}
		columns := strings.Split(v, "\t")
		row := TmpRow7{}
		index := 0
		for j, col := range columns {
			index = translate[j]
			switch {
			case index == 1:
				row.Str1 = col
			case index == 2:
				row.Str2 = col
			case index == 3:
				row.Str3 = col
			case index == 4:
				row.Str4 = col
			case index == 5:
				row.Str5 = col
			case index == 6:
				row.Str6 = col
			case index == 7:
				row.Str7 = col
			case index >= 7 && index <= 14:
				tmpInt, err := strconv.ParseInt(col, 10, 64)
				if err != nil {
					tmpInt = 0
				}
				switch {
				case index == 8:
					row.Int1 = tmpInt
				case index == 9:
					row.Int2 = tmpInt
				case index == 10:
					row.Int3 = tmpInt
				case index == 11:
					row.Int4 = tmpInt
				case index == 12:
					row.Int5 = tmpInt
				case index == 13:
					row.Int6 = tmpInt
				case index == 14:
					row.Int7 = tmpInt
				}
			case index >= 15 && index <= 21:
				tmpFloat, err := strconv.ParseFloat(col, 64)
				if err != nil {
					tmpFloat = 0
				}
				switch {
				case index == 15:
					row.Float1 = tmpFloat
				case index == 16:
					row.Float2 = tmpFloat
				case index == 17:
					row.Float3 = tmpFloat
				case index == 18:
					row.Float4 = tmpFloat
				case index == 19:
					row.Float5 = tmpFloat
				case index == 20:
					row.Float6 = tmpFloat
				case index == 21:
					row.Float7 = tmpFloat
				}

			}
		}

		result = append(result, row)
	}
	return result
}

func Base64Encode(msg string) string {
	return base64.StdEncoding.EncodeToString([]byte(msg))
}

func Base64Decode(encoded string) string {
	decoded, err := base64.StdEncoding.DecodeString(encoded)
	if err != nil {
		return ""
	}
	return string(decoded)

}
<|endoftext|>


<commit_before>// Package goclient provides functions to interact with the preact-rpc server.
// The Connect function needs to be called first, which will open a socket to
// the server. Then any RenderComponent calls can be made.
package goclient

import (
  "bufio"
  "bytes"
  "encoding/json"
  "fmt"
  "net"
  "sync"
)

// Buffer data marker
const data_end_marker = "\r\n."

// The connection to the RPC server
var conn net.Conn
var shouldConnect bool
var connNet string
var connAddr string

// Request ID
var reqId int = 0

// Mutex to synchronize render calls
var mu sync.Mutex

// Split function for the Scanner to create response tokens from the
// connection read buffer.
func split(data []byte, atEOF bool) (advance int, token []byte, err error) {
  idx := bytes.Index(data, []byte(data_end_marker))
  if idx > -1 {
    return idx + len(data_end_marker), data[:idx], nil
  }

  // If we reach EOF before getting the data end marker we have an error.
  if atEOF {
    return 0, nil, fmt.Errorf("Bad output")
  }

  // Data end marker not found, read more bytes.
  return 0, nil, nil
}

// An RpcResponse is returned for a RenderComponent() call.
// It consists of an Id, the rendered HTML, the html for hydrating
// the redux store and an error message from the server.
// The Id is the same value sent in the request, and is used to
// match the response to the request.
type RpcResponse struct {
  Id int `json:"id"`
  Html string `json:"html"`
  HydrateHtml string `json:"hydrate_html"`
  Error string `json:"error"`
}

// Error types
var (
  ErrConnectionWrite = fmt.Errorf("Could not write to connection")
)

// Connect to RPC server.
// The parameters are the same as that of net.Dial(), and depend
// on where the preact-rpc server is listening.
func Connect(network string, address string) {
  connNet = network
  connAddr = address
  shouldConnect = true
}

// Send render request for a component and associated props, and get HTML response.
// The Connect() function must be called before calling this function.
func RenderComponent(componentName string, storeName *string, props interface{}) (*RpcResponse, error) {
  mu.Lock()
  defer mu.Unlock()

  var err error
  if shouldConnect {
    conn, err = net.Dial(connNet, connAddr)
    if err != nil {
      return nil, err
    }
    shouldConnect = false
  }

  // Increment request ID
  reqId += 1

  // Render request to RPC server.
  payload := map[string]interface{}{
    "id": reqId,
    "component": componentName,
  }

  // Add props if there are any
  if props != nil {
    payload["props"] = props
  }

  // Add store name if provided
  if storeName != nil {
    payload["store"] = *storeName
  }

  jsonPayload, err := json.Marshal(payload)
  if err != nil {
    return nil, err
  }

  if _, err := fmt.Fprintf(conn, string(jsonPayload) + data_end_marker); err != nil {
    shouldConnect = true
    return nil, ErrConnectionWrite
  }

  // Parse JSON response.
  scanner := bufio.NewScanner(bufio.NewReader(conn))
  scanner.Split(split)
  scanner.Scan()
  jsonBlob := scanner.Bytes()

  // Error reading buffer ...
  if err := scanner.Err(); err != nil {
    return nil, err
  }
  // Error parsing JSON
  var resp RpcResponse
  if err := json.Unmarshal(jsonBlob, &resp); err != nil {
    return nil, err
  }

  // Add hydration html if store was provided
  if storeName != nil && props != nil {
    jsonProps, _ := json.Marshal(props)

    resp.HydrateHtml = fmt.Sprintf(`
      <div
        class='preact-rpc-store'
        data-props='%s'
        data-store-name='%s'
        style='display: none'>
      </div>
    `, jsonProps, *storeName)
  }

  return &resp, nil
}
<commit_msg>factor out goclient connection to network<commit_after>// Package goclient provides functions to interact with the preact-rpc server.
// The Connect function needs to be called first, which will open a socket to
// the server. Then any RenderComponent calls can be made.
package goclient

import (
  "bufio"
  "bytes"
  "encoding/json"
  "fmt"
  "net"
  "sync"
)

// Buffer data marker
const data_end_marker = "\r\n."

// The connection to the RPC server
var conn net.Conn
var shouldConnect bool
var connNet string
var connAddr string

// Request ID
var reqId int = 0

// Mutex to synchronize render calls
var mu sync.Mutex

// Split function for the Scanner to create response tokens from the
// connection read buffer.
func split(data []byte, atEOF bool) (advance int, token []byte, err error) {
  idx := bytes.Index(data, []byte(data_end_marker))
  if idx > -1 {
    return idx + len(data_end_marker), data[:idx], nil
  }

  // If we reach EOF before getting the data end marker we have an error.
  if atEOF {
    return 0, nil, fmt.Errorf("Bad output")
  }

  // Data end marker not found, read more bytes.
  return 0, nil, nil
}

// An RpcResponse is returned for a RenderComponent() call.
// It consists of an Id, the rendered HTML, the html for hydrating
// the redux store and an error message from the server.
// The Id is the same value sent in the request, and is used to
// match the response to the request.
type RpcResponse struct {
  Id int `json:"id"`
  Html string `json:"html"`
  HydrateHtml string `json:"hydrate_html"`
  Error string `json:"error"`
}

// Error types
var (
  ErrConnectionWrite = fmt.Errorf("Could not write to connection")
)

// Connect to RPC server.
// The parameters are the same as that of net.Dial(), and depend
// on where the preact-rpc server is listening.
func Connect(network string, address string) {
  connNet = network
  connAddr = address
  shouldConnect = true
}

func connectToNetwork() error {
  var err error
  conn, err = net.Dial(connNet, connAddr)
  return err
}

// Send render request for a component and associated props, and get HTML response.
// The Connect() function must be called before calling this function.
func RenderComponent(componentName string, storeName *string, props interface{}) (*RpcResponse, error) {
  mu.Lock()
  defer mu.Unlock()

  if shouldConnect {
    if err := connectToNetwork(); err != nil {
      return nil, err
    }
    shouldConnect = false
  }

  // Increment request ID
  reqId += 1

  // Render request to RPC server.
  payload := map[string]interface{}{
    "id": reqId,
    "component": componentName,
  }

  // Add props if there are any
  if props != nil {
    payload["props"] = props
  }

  // Add store name if provided
  if storeName != nil {
    payload["store"] = *storeName
  }

  jsonPayload, err := json.Marshal(payload)
  if err != nil {
    return nil, err
  }

  if _, err := fmt.Fprintf(conn, string(jsonPayload) + data_end_marker); err != nil {
    // We'll try reconnecting in the next call
    shouldConnect = true
    return nil, ErrConnectionWrite
  }

  // Parse JSON response.
  scanner := bufio.NewScanner(bufio.NewReader(conn))
  scanner.Split(split)
  scanner.Scan()
  jsonBlob := scanner.Bytes()

  // Error reading buffer ...
  if err := scanner.Err(); err != nil {
    return nil, err
  }
  // Error parsing JSON
  var resp RpcResponse
  if err := json.Unmarshal(jsonBlob, &resp); err != nil {
    return nil, err
  }

  // Add hydration html if store was provided
  if storeName != nil && props != nil {
    jsonProps, _ := json.Marshal(props)

    resp.HydrateHtml = fmt.Sprintf(`
      <div
        class='preact-rpc-store'
        data-props='%s'
        data-store-name='%s'
        style='display: none'>
      </div>
    `, jsonProps, *storeName)
  }

  return &resp, nil
}
<|endoftext|>